fusion-bench 0.2.25__py3-none-any.whl → 0.2.26__py3-none-any.whl

fusion_bench/method/__init__.py

@@ -70,6 +70,7 @@ _import_structure = {
         "IsotropicMergingInCommonSubspace",
     ],
     "opcm": ["OPCMForCLIP"],
+    "dop": ["ContinualDOPForCLIP"],
     "gossip": [
         "CLIPLayerWiseGossipAlgorithm",
         "CLIPTaskWiseGossipAlgorithm",
@@ -212,6 +213,7 @@ if TYPE_CHECKING:
     from .model_recombination import ModelRecombinationAlgorithm
     from .model_stock import ModelStock
     from .opcm import OPCMForCLIP
+    from .dop import ContinualDOPForCLIP
     from .pruning import (
         MagnitudeDiffPruningAlgorithm,
         MagnitudePruningForLlama,

fusion_bench/method/classification/clip_finetune.py

@@ -5,8 +5,8 @@ Fine-tune CLIP-ViT-B/32:
 
 ```bash
 fusion_bench \
-    method=clip_finetune \
-    modelpool=clip-vit-base-patch32_mtl \
+    method=classification/clip_finetune \
+    modelpool=CLIPVisionModelPool/clip-vit-base-patch32_mtl \
     taskpool=dummy
 ```
 
@@ -15,12 +15,14 @@ Fine-tune CLIP-ViT-L/14 on eight GPUs with a per-device per-task batch size of 2
 ```bash
 fusion_bench \
     fabric.devices=8 \
-    method=clip_finetune \
+    method=classification/clip_finetune \
         method.batch_size=2 \
-    modelpool=clip-vit-base-patch32_mtl \
+    modelpool=CLIPVisionModelPool/clip-vit-base-patch32_mtl \
         modelpool.models.0.path=openai/clip-vit-large-patch14 \
     taskpool=dummy
 ```
+
+See `examples/clip_finetune` for more details.
 """
 
 import os

fusion_bench/method/dop/__init__.py

@@ -0,0 +1 @@
+from .dop import ContinualDOPForCLIP

fusion_bench/method/dop/dop.py

@@ -0,0 +1,366 @@
+"""
+Continual Model Merging without Data: Dual Projections for Balancing Stability and Plasticity. NeurIPS, 2025.
+
+
+Example:
+
+fusion_bench \
+    method=dop/dop \
+    modelpool=CLIPVisionModelPool/clip-vit-base-patch32_TA8_model_only \
+    taskpool=CLIPVisionModelTaskPool/clip-vit-classification_TA8
+"""
+
+import logging
+import os
+import random
+from copy import deepcopy
+from pathlib import Path
+from typing import Dict, List, Literal, Optional, Tuple, cast
+
+import lightning as L
+import numpy as np
+import torch
+from omegaconf import DictConfig
+from torch import Tensor, nn
+from torch.autograd import Variable
+from tqdm.auto import tqdm
+from transformers import CLIPVisionModel
+
+from fusion_bench import BaseAlgorithm, BaseModelPool, auto_register_config
+from fusion_bench.method.simple_average import simple_average
+from fusion_bench.mixins import LightningFabricMixin
+from fusion_bench.taskpool import CLIPVisionModelTaskPool
+from fusion_bench.utils import seed_everything_by_time
+from fusion_bench.utils.json import save_to_json
+
+from .min_norm_solvers import MinNormSolver, gradient_normalizers
+from .utils import is_leaf_module, svd
+
+log = logging.getLogger(__name__)
+
+
+@auto_register_config
+class ContinualDOPForCLIP(BaseAlgorithm, LightningFabricMixin):
+
+    def __init__(
+        self,
+        seed: Optional[int] = None,
+        shuffle_order: bool = False,
+        save_on_every_step: bool = True,
+        evaluate_on_every_step: bool = False,
+        lr: float = 1e-4,
+        num_steps: int = 200,
+        mgda: bool = True,
+        ema: bool = True,
+        ema_beta: float = 0.99,
+        alpha: float = None,
+        svd_epsilon: float = 1.0,
+        svd_proj_space: str = "uv",
+        **kwargs,
+    ):
+        self.lr = lr
+        self.num_steps = num_steps
+        self.mgda = mgda
+        self.ema = ema
+        self.ema_beta = ema_beta
+        self.alpha = alpha
+        self.svd_epsilon = svd_epsilon
+        self.svd_proj_space = svd_proj_space
+        self.seed = seed
+        self.shuffle_order = shuffle_order
+        self.save_on_every_step = save_on_every_step
+        self.evaluate_on_every_step = evaluate_on_every_step
+
+        assert (
+            self.svd_epsilon >= 0 and self.svd_epsilon <= 1
+        ), "The svd_epsilon should be in the range of [0, 1]"
+        assert (
+            self.alpha >= 0 and self.alpha <= 1
+        ), "The alpha should be in the range of [0, 1]"
+        super().__init__(**kwargs)
+
+    def print_params(self, pretrained_model):
+        total_params = 0
+        linear_params = 0
+        linear_weight_params = 0
+        for module_name, module in pretrained_model.named_modules():
+            if not is_leaf_module(module):
+                continue
+            if isinstance(module, nn.Linear):
+                linear_params += sum(p.numel() for n, p in module.named_parameters())
+                linear_weight_params += sum(
+                    p.numel() for n, p in module.named_parameters() if "weight" in n
+                )
+            total_params += sum(p.numel() for p in module.parameters())
+
+        linear_ratio = linear_params / total_params * 100
+        linear_weight_ratio = linear_weight_params / total_params * 100
+        print(f"Total Parameters: {total_params}")
+        print(f"Linear Parameters: {linear_params}")
+        print(f"Linear Weight Parameters: {linear_weight_params}")
+        print(f"Linear Ratio: {linear_ratio:.2f}%")
+        print(f"Linear Weight Ratio: {linear_weight_ratio:.2f}%")
+
+    def run(self, modelpool: BaseModelPool):
+        if self.seed is not None:
+            L.seed_everything(self.seed)
+        else:
+            seed_everything_by_time(self.fabric)
+
+        # get the model names, shuffle if needed
+        # the model names will be saved to the log directory as `model_names.json`
+        model_names = modelpool.model_names
+        if self.shuffle_order:
+            random.shuffle(model_names)
+        if self.log_dir is not None:
+            save_to_json(model_names, os.path.join(self.log_dir, "model_names.json"))
+
+        if self.evaluate_on_every_step:
+            """Configuration for the test datasets"""
+            self.taskpool = cast(CLIPVisionModelTaskPool, self._program.taskpool)
+            self._test_datasets = deepcopy(self.taskpool._test_datasets)
+
+        pretrained_model = modelpool.load_pretrained_model()
+
+        merged_model = None
+        for model_idx, model_name in enumerate(model_names):
+            print(
+                f"--------- Optimizing {model_idx + 1}/{len(model_names)}-th with {model_name} ---------"
+            )
+            if model_idx == 0:
+                merged_model = modelpool.load_model(model_names[0])
+            else:
+                merged_model = self._layer_wise_optimize(
+                    model_names=["merged", model_name],
+                    pretrained_model=deepcopy(pretrained_model),
+                    finetuned_models={
+                        "merged": merged_model,
+                        model_name: modelpool.load_model(model_name),
+                    },
+                    model_idx=model_idx,
+                )
+
+            if self.save_on_every_step:
+                self.save_merged_model(merged_model, model_idx)
+
+            if self.evaluate_on_every_step:
+                self.taskpool._is_setup = False
+                self.taskpool._test_datasets = DictConfig(
+                    {n: self._test_datasets[n] for n in model_names[: model_idx + 1]}
+                )
+                report = self.taskpool.evaluate(deepcopy(merged_model))
+                save_to_json(report, Path(self.log_dir) / f"report_{model_idx}.json")
+
+        return merged_model
+
+    def _layer_wise_optimize(
+        self,
+        model_names: List[str],
+        pretrained_model: nn.Module,
+        finetuned_models: Dict[str, nn.Module],
+        model_idx: int,
+    ):
+        time_cost = []
+        for module_name, module in pretrained_model.named_modules():
+            if not is_leaf_module(module):
+                continue
+
+            if isinstance(module, nn.Linear):
+                if module.weight.requires_grad:
+                    import time
+
+                    start_time = time.time()
+                    merged_weight = self._optimize_weight(
+                        module.weight,
+                        {
+                            model_name: finetuned_models[model_name]
+                            .get_submodule(module_name)
+                            .weight
+                            for model_name in model_names
+                        },
+                        module_name,
+                        model_idx,
+                    )
+                    end_time = time.time()
+                    time_cost.append(end_time - start_time)
+                    module.weight.data = merged_weight.data
+                else:
+                    module.weight.data = simple_average(
+                        [
+                            finetuned_models[model_name]
+                            .get_submodule(module_name)
+                            .weight
+                            for model_name in model_names
+                        ]
+                    )
+                if module.bias is not None:
+                    module.bias.data = simple_average(
+                        [
+                            finetuned_models[model_name].get_submodule(module_name).bias
+                            for model_name in model_names
+                        ]
+                    )
+            else:
+                simple_average(
+                    [
+                        finetuned_models[model_name].get_submodule(module_name)
+                        for model_name in model_names
+                    ],
+                    base_module=module,
+                )
+
+        return pretrained_model
+
+    def _optimize_weight(
+        self,
+        pretrained_weight: Tensor,
+        finetuned_weights: Dict[str, Tensor],
+        module_name: str,
+        model_idx: int,
+    ):
+        assert (
+            self.fabric.world_size == 1
+        ), "This algorithm is not currently supported in distributed training"
+
+        pretrained_weight = self.fabric.to_device(pretrained_weight.detach())
+        finetuned_weights = {
+            model_name: self.fabric.to_device(finetuned_weight.detach())
+            for model_name, finetuned_weight in finetuned_weights.items()
+        }
+
+        merged_weight = self.fabric.to_device(
+            nn.Parameter(
+                simple_average(
+                    [
+                        finetuned_weight.detach()
+                        for finetuned_weight in finetuned_weights.values()
+                    ]
+                ),
+                requires_grad=True,
+            )
+        )
+
+        # Compute SVD of the difference between the finetuned and pretrained weights
+        proj_u_dict = {}
+        proj_v_dict = {}
+        proj_s_dict = {}
+        for i, finetuned_weight in enumerate(finetuned_weights.values()):
+            finetuned_tv = finetuned_weight - pretrained_weight
+            u, s, v = svd(finetuned_tv, full_matrices=True)
+            epsilon = 1.0 if self.svd_epsilon > 1.0 else self.svd_epsilon
+            cumsum_ratio = s.cumsum(dim=0) / s.sum()
+            split_rank = torch.searchsorted(cumsum_ratio, epsilon).item()
+            u_main = u[:, :split_rank]
+            v_main = v[:, :split_rank]
+            s_main = s[:split_rank]
+            proj_u_dict[i] = u_main
+            proj_v_dict[i] = v_main
+            proj_s_dict[i] = s_main
+
+        if self.mgda:
+            if self.ema:
+                ema_sol = [self.alpha, 1 - self.alpha]
+            # This is multiple-gradient descent algorithm (MGDA) optimization
+            optimizer = torch.optim.Adam([merged_weight], lr=self.lr)
+            all_losses = [[], []]
+            all_alphas = [[], []]
+            for step_idx in tqdm(
+                range(self.num_steps), desc=f"Optimizing {module_name} weight"
+            ):
+                # Scaling the loss functions based on the algorithm choice
+                loss_data = {}
+                grads = {}
+                for i, finetuned_weight in enumerate(finetuned_weights.values()):
+                    proj_u = proj_u_dict[i]
+                    proj_v = proj_v_dict[i]
+                    proj_s = proj_s_dict[i]
+                    delta_tv = merged_weight - finetuned_weight
+                    loss_i = self.cal_loss_i(delta_tv, proj_s, proj_u, proj_v)
+                    loss_data[i] = float(loss_i.data)
+
+                    all_losses[i].append(float(loss_i.data))
+
+                    optimizer.zero_grad()
+                    loss_i.backward()
+                    grads[i] = Variable(
+                        merged_weight.grad.data.clone(), requires_grad=False
+                    )
+
+                # Normalize all gradients
+                gn = gradient_normalizers(
+                    grads=grads, losses=loss_data, normalization_type="loss"
+                )
+                for i, _ in enumerate(finetuned_weights.values()):
+                    grads[i] = grads[i] / float(gn[i])
+
+                # Frank-Wolfe iteration to compute scales.
+                sol, min_norm = MinNormSolver.find_min_norm_element(
+                    [[grads[i]] for i in range(len(finetuned_weights.values()))]
+                )
+
+                if self.ema:
+                    ema_sol = [
+                        self.ema_beta * ema_sol[i] + (1 - self.ema_beta) * float(sol[i])
+                        for i in range(len(sol))
+                    ]
+                    sol = ema_sol
+                    all_alphas[0].append(ema_sol[0])
+                    all_alphas[1].append(ema_sol[1])
+
+                # Scaled back-propagation
+                loss = 0
+                for i, finetuned_weight in enumerate(finetuned_weights.values()):
+                    # Comptue gradients of each loss function wrt parameters
+                    proj_u = proj_u_dict[i]
+                    proj_v = proj_v_dict[i]
+                    proj_s = proj_s_dict[i]
+                    delta_tv = merged_weight - finetuned_weight
+                    loss_i = self.cal_loss_i(delta_tv, proj_s, proj_u, proj_v)
+                    loss += float(sol[i]) * loss_i
+
+                optimizer.zero_grad()
+                loss.backward()
+                optimizer.step()
+
+        else:
+            # This is a naive weighted optimization
+            optimizer = torch.optim.Adam([merged_weight], lr=self.lr)
+            for step_idx in tqdm(
+                range(self.num_steps), desc=f"Optimizing {module_name} weight"
+            ):
+                loss = 0
+                for i, finetuned_weight in enumerate(finetuned_weights.values()):
+                    proj_u = proj_u_dict[i]
+                    proj_v = proj_v_dict[i]
+                    proj_s = proj_s_dict[i]
+                    delta_tv = merged_weight - finetuned_weight
+                    loss_i = self.cal_loss_i(delta_tv, proj_s, proj_u, proj_v)
+                    loss += self.alpha * loss_i if i == 0 else (1 - self.alpha) * loss_i
+
+                optimizer.zero_grad()
+                loss.backward()
+                optimizer.step()
+
+        return merged_weight.detach().cpu()
+
+    def cal_loss_i(self, delta_tv, proj_s, proj_u, proj_v):
+        proj_delta_1 = torch.diag(proj_s) @ proj_u.T @ delta_tv
+        proj_delta_2 = delta_tv @ proj_v @ torch.diag(proj_s)
+        loss_i_u = torch.linalg.matrix_norm(proj_delta_1, ord="fro") ** 2
+        loss_i_v = torch.linalg.matrix_norm(proj_delta_2, ord="fro") ** 2
+        if self.svd_proj_space == "uv":
+            loss_i = loss_i_u + loss_i_v
+        elif self.svd_proj_space == "u":
+            loss_i = loss_i_u
+        elif self.svd_proj_space == "v":
+            loss_i = loss_i_v
+        else:
+            raise ValueError("Invalid svd_proj_space")
+
+        return loss_i
+
+    def save_merged_model(self, merged_model: CLIPVisionModel, step: int):
+        os.makedirs(Path(self.log_dir) / "checkpoints", exist_ok=True)
+        merged_model.save_pretrained(
+            Path(self.log_dir) / "checkpoints" / f"merged_model_{step}"
+        )

fusion_bench/method/dop/min_norm_solvers.py

@@ -0,0 +1,227 @@
+# This code is from
+# Multi-Task Learning as Multi-Objective Optimization
+# Ozan Sener, Vladlen Koltun
+# Neural Information Processing Systems (NeurIPS) 2018
+# https://github.com/intel-isl/MultiObjectiveOptimization
+from typing import Union
+
+import numpy as np
+import torch
+
+
+def np_sum(x: Union[torch.Tensor, np.ndarray]) -> float:
+    if isinstance(x, torch.Tensor):
+        return x.sum().item()
+    return np.sum(x)
+
+
+def to_numpy(x: Union[torch.Tensor, np.ndarray]) -> np.ndarray:
+    if isinstance(x, torch.Tensor):
+        return x.detach().cpu().numpy()
+    return x
+
+
+class MinNormSolver:
+    MAX_ITER = 250
+    STOP_CRIT = 1e-5
+
+    def _min_norm_element_from2(v1v1, v1v2, v2v2):
+        """
+        Analytical solution for min_{c} |cx_1 + (1-c)x_2|_2^2
+        d is the distance (objective) optimzed
+        v1v1 = <x1,x1>
+        v1v2 = <x1,x2>
+        v2v2 = <x2,x2>
+        """
+        if v1v2 >= v1v1:
+            # Case: Fig 1, third column
+            gamma = 0.999
+            cost = v1v1
+            return gamma, cost
+        if v1v2 >= v2v2:
+            # Case: Fig 1, first column
+            gamma = 0.001
+            cost = v2v2
+            return gamma, cost
+        # Case: Fig 1, second column
+        gamma = -1.0 * ((v1v2 - v2v2) / (v1v1 + v2v2 - 2 * v1v2))
+        cost = v2v2 + gamma * (v1v2 - v2v2)
+        return gamma, cost
+
+    def _min_norm_2d(vecs, dps):
+        R"""
+        Find the minimum norm solution as combination of two points
+        This is correct only in 2D
+        ie. min_c |\sum c_i x_i|_2^2 st. \sum c_i = 1 , 1 >= c_1 >= 0 for all i, c_i + c_j = 1.0 for some i, j
+        """
+        dmin = 1e8
+        for i in range(len(vecs)):
+            for j in range(i + 1, len(vecs)):
+                if (i, j) not in dps:
+                    dps[(i, j)] = 0.0
+                    for k in range(len(vecs[i])):
+                        dps[(i, j)] += (
+                            torch.mul(vecs[i][k], vecs[j][k]).sum().data.cpu()
+                        )
+                    dps[(j, i)] = dps[(i, j)]
+                if (i, i) not in dps:
+                    dps[(i, i)] = 0.0
+                    for k in range(len(vecs[i])):
+                        dps[(i, i)] += (
+                            torch.mul(vecs[i][k], vecs[i][k]).sum().data.cpu()
+                        )
+                if (j, j) not in dps:
+                    dps[(j, j)] = 0.0
+                    for k in range(len(vecs[i])):
+                        dps[(j, j)] += (
+                            torch.mul(vecs[j][k], vecs[j][k]).sum().data.cpu()
+                        )
+                c, d = MinNormSolver._min_norm_element_from2(
+                    dps[(i, i)], dps[(i, j)], dps[(j, j)]
+                )
+                if d < dmin:
+                    dmin = d
+                    sol = [(i, j), c, d]
+        return sol, dps
+
+    def _projection2simplex(y):
+        R"""
+        Given y, it solves argmin_z |y-z|_2 st \sum z = 1 , 1 >= z_i >= 0 for all i
+        """
+        m = len(y)
+        sorted_y = np.flip(np.sort(y), axis=0)
+        tmpsum = 0.0
+        tmax_f = (np.sum(y) - 1.0) / m
+        for i in range(m - 1):
+            tmpsum += sorted_y[i]
+            tmax = (tmpsum - 1) / (i + 1.0)
+            if tmax > sorted_y[i + 1]:
+                tmax_f = tmax
+                break
+        return np.maximum(y - tmax_f, np.zeros(y.shape))
+
+    def _next_point(cur_val, grad, n):
+        proj_grad = grad - (np.sum(grad) / n)
+        tm1 = -1.0 * cur_val[proj_grad < 0] / proj_grad[proj_grad < 0]
+        tm2 = (1.0 - cur_val[proj_grad > 0]) / (proj_grad[proj_grad > 0])
+
+        skippers = np_sum(tm1 < 1e-7) + np_sum(tm2 < 1e-7)
+        t = 1
+        if len(tm1[tm1 > 1e-7]) > 0:
+            t = np.min(to_numpy(tm1[tm1 > 1e-7]))
+        if len(tm2[tm2 > 1e-7]) > 0:
+            t = min(t, np.min(to_numpy(tm2[tm2 > 1e-7])))
+
+        next_point = proj_grad * t + to_numpy(cur_val)
+        next_point = MinNormSolver._projection2simplex(next_point)
+        return next_point
+
+    def find_min_norm_element(vecs):
+        R"""
+        Given a list of vectors (vecs), this method finds the minimum norm element in the convex hull
+        as min |u|_2 st. u = \sum c_i vecs[i] and \sum c_i = 1.
+        It is quite geometric, and the main idea is the fact that if d_{ij} = min |u|_2 st u = c x_i + (1-c) x_j; the solution lies in (0, d_{i,j})
+        Hence, we find the best 2-task solution, and then run the projected gradient descent until convergence
+        """
+        # Solution lying at the combination of two points
+        dps = {}
+        init_sol, dps = MinNormSolver._min_norm_2d(vecs, dps)
+
+        n = len(vecs)
+        sol_vec = np.zeros(n)
+        sol_vec[init_sol[0][0]] = init_sol[1]
+        sol_vec[init_sol[0][1]] = 1 - init_sol[1]
+
+        if n < 3:
+            # This is optimal for n=2, so return the solution
+            return sol_vec, init_sol[2]
+
+        iter_count = 0
+
+        grad_mat = np.zeros((n, n))
+        for i in range(n):
+            for j in range(n):
+                grad_mat[i, j] = dps[(i, j)]
+
+        while iter_count < MinNormSolver.MAX_ITER:
+            grad_dir = -1.0 * np.dot(grad_mat, sol_vec)
+            new_point = MinNormSolver._next_point(sol_vec, grad_dir, n)
+            # Re-compute the inner products for line search
+            v1v1 = 0.0
+            v1v2 = 0.0
+            v2v2 = 0.0
+            for i in range(n):
+                for j in range(n):
+                    v1v1 += sol_vec[i] * sol_vec[j] * dps[(i, j)]
+                    v1v2 += sol_vec[i] * new_point[j] * dps[(i, j)]
+                    v2v2 += new_point[i] * new_point[j] * dps[(i, j)]
+            nc, nd = MinNormSolver._min_norm_element_from2(v1v1, v1v2, v2v2)
+            new_sol_vec = nc * sol_vec + (1 - nc) * new_point
+            change = new_sol_vec - sol_vec
+            if np_sum(np.abs(change)) < MinNormSolver.STOP_CRIT:
+                return sol_vec, nd
+            sol_vec = new_sol_vec
+
+    def find_min_norm_element_FW(vecs):
+        R"""
+        Given a list of vectors (vecs), this method finds the minimum norm element in the convex hull
+        as min |u|_2 st. u = \sum c_i vecs[i] and \sum c_i = 1.
+        It is quite geometric, and the main idea is the fact that if d_{ij} = min |u|_2 st u = c x_i + (1-c) x_j; the solution lies in (0, d_{i,j})
+        Hence, we find the best 2-task solution, and then run the Frank Wolfe until convergence
+        """
+        # Solution lying at the combination of two points
+        dps = {}
+        init_sol, dps = MinNormSolver._min_norm_2d(vecs, dps)
+
+        n = len(vecs)
+        sol_vec = np.zeros(n)
+        sol_vec[init_sol[0][0]] = init_sol[1]
+        sol_vec[init_sol[0][1]] = 1 - init_sol[1]
+
+        if n < 3:
+            # This is optimal for n=2, so return the solution
+            return sol_vec, init_sol[2]
+
+        iter_count = 0
+
+        grad_mat = np.zeros((n, n))
+        for i in range(n):
+            for j in range(n):
+                grad_mat[i, j] = dps[(i, j)]
+
+        while iter_count < MinNormSolver.MAX_ITER:
+            t_iter = np.argmin(np.dot(grad_mat, sol_vec))
+
+            v1v1 = np.dot(sol_vec, np.dot(grad_mat, sol_vec))
+            v1v2 = np.dot(sol_vec, grad_mat[:, t_iter])
+            v2v2 = grad_mat[t_iter, t_iter]
+
+            nc, nd = MinNormSolver._min_norm_element_from2(v1v1, v1v2, v2v2)
+            new_sol_vec = nc * sol_vec
+            new_sol_vec[t_iter] += 1 - nc
+
+            change = new_sol_vec - sol_vec
+            if np_sum(np.abs(change)) < MinNormSolver.STOP_CRIT:
+                return sol_vec, nd
+            sol_vec = new_sol_vec
+
+
+def gradient_normalizers(grads, losses, normalization_type):
+    gn = {}
+    if normalization_type == "l2":
+        for t in grads:
+            gn[t] = np.sqrt(np.sum([gr.pow(2).sum().data.cpu() for gr in grads[t]]))
+    elif normalization_type == "loss":
+        for t in grads:
+            gn[t] = losses[t]
+    elif normalization_type == "loss+":
+        for t in grads:
+            gn[t] = losses[t] * np.sqrt(
+                np.sum([gr.pow(2).sum().data.cpu() for gr in grads[t]])
+            )
+    elif normalization_type == "none":
+        for t in grads:
+            gn[t] = 1.0
+    else:
+        print("ERROR: Invalid Normalization Type")
+    return gn

fusion_bench/method/dop/utils.py

@@ -0,0 +1,73 @@
+from typing import Tuple
+
+import torch
+from torch import Tensor, nn
+
+from fusion_bench.utils.parameters import state_dict_to_vector
+from fusion_bench.utils.state_dict_arithmetic import state_dict_sub
+
+
+def _svd(w: Tensor, full_matrices=True) -> Tuple[Tensor, Tensor, Tensor]:
+    """
+    Perform Singular Value Decomposition (SVD) on a tensor.
+
+    Args:
+        w (Tensor): The input tensor.
+        full_matrices (bool): Whether to compute the full-sized U and V matrices.
+
+    Returns:
+        Tuple[Tensor, Tensor, Tensor]: The U, S, and V matrices from SVD.
+    """
+    u, s, vh = torch.linalg.svd(
+        w, full_matrices=full_matrices, driver="gesvd" if w.is_cuda else None
+    )
+    v = vh.T
+    return u, s, v
+
+
+def svd(
+    w: Tensor, full_matrices=True, accelerator=None
+) -> Tuple[Tensor, Tensor, Tensor]:
+    """
+    Perform SVD on a tensor, optionally using a specified accelerator.
+
+    Args:
+        w (Tensor): The input tensor.
+        full_matrices (bool): Whether to compute the full-sized U and V matrices.
+        accelerator (str): The device to perform the computation on.
+
+    Returns:
+        Tuple[Tensor, Tensor, Tensor]: The U, S, and V matrices from SVD.
+    """
+    if accelerator is None:
+        return _svd(w, full_matrices=full_matrices)
+    original_device = w.device
+    w = w.to(accelerator)
+    u, s, v = _svd(w)
+    return u.to(original_device), s.to(original_device), v.to(original_device)
+
+
+def frobenius_inner_product(w1: Tensor, w2: Tensor) -> Tensor:
+    return torch.trace(w1.T @ w2)
+
+
+def is_leaf_module(module: nn.Module) -> bool:
+    return len(list(module.children())) == 0
+
+
+def get_task_vector_norm(model: nn.Module, pretrained_model: nn.Module) -> Tensor:
+    """
+    Get the vector norm of the task model.
+
+    Args:
+        model (nn.Module): The task model.
+        pretrained_model (nn.Module): The pretrained model.
+
+    Returns:
+        Tensor: The vector norm of the task model.
+    """
+    return torch.linalg.norm(
+        state_dict_to_vector(
+            state_dict_sub(model.state_dict(), pretrained_model.state_dict())
+        )
+    )

{fusion_bench-0.2.25.dist-info → fusion_bench-0.2.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fusion_bench
-Version: 0.2.25
+Version: 0.2.26
 Summary: A Comprehensive Benchmark of Deep Model Fusion
 Author-email: Anke Tang <tang.anke@foxmail.com>
 Project-URL: Repository, https://github.com/tanganke/fusion_bench
@@ -82,7 +82,13 @@ Model merging has emerged as a promising approach for multi-task learning (MTL),
 </details>
 
 <details>
-  <summary>Anke Tang, et al. Merging Models on the Fly Without Retraining: A Sequential Approach to Scalable Continual Model Merging. Jan 2025. https://arxiv.org/pdf/2501.09522</summary>
+  <summary>Enneng Yang, et al. Continual Model Merging without Data: Dual Projections for Balancing Stability and Plasticity. NeurIPS 2025. https://github.com/EnnengYang/DOP</summary>
+
+Model merging integrates multiple expert models with diverse capabilities into a unified framework, facilitating collaborative learning. However, most existing methods assume simultaneous access to all models, which is often impractical in real-world scenarios where models are received sequentially. While some studies have investigated continual model merging (CMM)--which involves sequentially merging multiple models--the challenge of balancing prior knowledge (stability) and incorporating new tasks (plasticity) remains unresolved. This paper, for the first time, formally defines the stability and plasticity of CMM from the perspective of orthogonal projection. Subsequently, we analyze the relationships among the spaces spanned by task data, historical gradients, and accumulated gradients. Building on this, we propose a data-free Dual Orthogonal Projection (DOP) method, which eliminates data dependence and mitigates interference between the merged model and models for old and new tasks by projecting their parameter differences onto their respective approximate data spaces. Finally, to solve potential conflicts between stability and plasticity, we reformulate DOP as a multi-objective optimization problem and employ a multi-gradient descent algorithm to obtain a Pareto-optimal solution. Extensive experiments across multiple architectures and task configurations validate that our approach significantly outperforms state-of-the-art CMM methods.
+</details>
+
+<details>
+  <summary>Anke Tang, et al. Merging Models on the Fly Without Retraining: A Sequential Approach to Scalable Continual Model Merging. NeurIPS 2025. Jan 2025. https://arxiv.org/pdf/2501.09522</summary>
 
 Deep model merging represents an emerging research direction that combines multiple fine-tuned models to harness their specialized capabilities across different tasks and domains. Current model merging techniques focus on merging all available models simultaneously, with weight interpolation-based methods being the predominant approaches. However, these conventional approaches are not well-suited for scenarios where models become available sequentially, and they often suffer from high memory requirements and potential interference between tasks. In this study, we propose a training-free projection-based continual merging method that processes models sequentially through orthogonal projections of weight matrices and adaptive scaling mechanisms. Our method operates by projecting new parameter updates onto subspaces orthogonal to existing merged parameter updates while using an adaptive scaling mechanism to maintain stable parameter distances, enabling efficient sequential integration of task-specific knowledge. Our approach maintains constant memory complexity to the number of models, minimizes interference between tasks through orthogonal projections, and retains the performance of previously merged models through adaptive task vector scaling. Extensive experiments on CLIP-ViT models demonstrate that our method achieves a 5-8% average accuracy improvement while maintaining robust performance in different task orderings.
 </details>

{fusion_bench-0.2.25.dist-info → fusion_bench-0.2.26.dist-info}/RECORD

@@ -48,7 +48,7 @@ fusion_bench/dataset/llama/stanford_shp.py,sha256=6ueXKnFXIBBobacU1h5WxGLZrSOtBk
 fusion_bench/dataset/llama/ultrachat.py,sha256=Go7WvrDAYnm184fdazHGRYLbSY6Xd7jrESyQeUJtOww,1736
 fusion_bench/dataset/llama/wikitext.py,sha256=9ZHR-nMfXRumd3o-PIj3n7B83YlVeqpGkZ2zJs2B-9Y,2883
 fusion_bench/dataset/llama/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fusion_bench/method/__init__.py,sha256=QOcRQ3AmUpSiDikH1tq-EcxakX7akFPOizcynTLmUwQ,9377
+fusion_bench/method/__init__.py,sha256=-r9Sz5haSc5X4eNvxGvHwowQfS2sLfZ21orUC5ae7ws,9454
 fusion_bench/method/base_algorithm.py,sha256=OnKSNPQ_nIdIWxryyblW_sko7uoEBN4lGh-eLkJ4kh4,9004
 fusion_bench/method/dummy.py,sha256=hb1y6LR_geRZ5eRgGwt5zJUcHYorCeIbs5i76CvurUc,1031
 fusion_bench/method/ensemble.py,sha256=Bjzqxt-tUp5cawT1jIhqKswN5QH3bkYbmuI4LS4uTG0,3619
@@ -77,7 +77,7 @@ fusion_bench/method/bitdelta/bitdelta_utils/binary_gemm_kernel.py,sha256=zC0w5cw
 fusion_bench/method/bitdelta/bitdelta_utils/data.py,sha256=LGEgv8o8glyyLLYh6Ur5h_sulxPFmy6i-xi-Ap1G-Wc,1052
 fusion_bench/method/bitdelta/bitdelta_utils/diff.py,sha256=o3ib5sgGDYLgnL8YTfX0YDc4Md6W9_gb03jzftTn5s4,4075
 fusion_bench/method/classification/__init__.py,sha256=byVJ574JQ_DUvsDv8S6ZM6BKAv4ZZ964Ej4btm0aC7k,867
-fusion_bench/method/classification/clip_finetune.py,sha256=QNOw9O-BTOVOsW7lzRu8L-UfbiBpsT_8tS6i6BpbVyA,15726
+fusion_bench/method/classification/clip_finetune.py,sha256=5q5Sr3eVVh8DfYdeSoGjwaKDksC8F2dY2r8Dl-wRaDg,15844
 fusion_bench/method/classification/continual_clip_finetune.py,sha256=OLhZKS-6aCnafevZkZYcNMKTWDDj3DATB27eZl_i8EY,11530
 fusion_bench/method/classification/image_classification_finetune.py,sha256=CPMpZvaULWaim01EvJJHlU4C6HQ16OCqZGoMvPBEWtY,8157
 fusion_bench/method/concrete_subspace/__init__.py,sha256=jJoFcjnQe-jvccsm9DuCXna378m9XBT9vV1fEZbdfR0,464
@@ -101,6 +101,10 @@ fusion_bench/method/doge_ta/__init__.py,sha256=dixO0i5fmhgC_W2_DAQ4PzYnkMCZX5D8t
 fusion_bench/method/doge_ta/clip_layer_wise_adamerging.py,sha256=4WPG2fhFw-u6oSoT-fBrP2K9YpX-MH-AotBL1DknfpA,1304
 fusion_bench/method/doge_ta/doge_ta.py,sha256=jrJF52JUBdrB3EGWaXJMFZE-v8syzZGr4smG6rEO74c,13790
 fusion_bench/method/doge_ta/layer_wise_adamerging.py,sha256=rLk3Nep5d6wMUNCp6q7pC7L0pfBvUwGBIuiGM7CQOf4,9780
+fusion_bench/method/dop/__init__.py,sha256=MD8c44ovLLJX_-v9t2SdLrvKLxVf8PijzFFNjJfvhpE,37
+fusion_bench/method/dop/dop.py,sha256=_wNjN1DSK27aKEyWVay61fqc7prwJ1uiv_3618_bQ20,14160
+fusion_bench/method/dop/min_norm_solvers.py,sha256=a7n2X0BE_YajlaUygyHV0yqW6-x5dTyZ5V0mt_Q69qE,8291
+fusion_bench/method/dop/utils.py,sha256=_q7yy3ENNFUh1qUd5J5DThRL4J1tIxEcknCO2AKmeYM,2102
 fusion_bench/method/expert_sparsity/__init__.py,sha256=nt7k5cKqA2Bax1aM93ODwsEuibZ_hdFgQsUos_8h2v8,271
 fusion_bench/method/expert_sparsity/mixtral/__init__.py,sha256=FyKDZIyYUnqvGIdJ5BS639UpzSBj11g28ATHs1Yczdk,545
 fusion_bench/method/expert_sparsity/mixtral/dynamic_skipping.py,sha256=zZa4IAKimFZMoxoQ_Oi7z2R9o5H6kxV2QTb0e-t9kDY,5665
@@ -480,7 +484,7 @@ fusion_bench/utils/plot/token_notebook.py,sha256=bsntXf46Zz_RavTxNiB9c3-KvHw7LFw
 fusion_bench/utils/strenum/__init__.py,sha256=id9ORi1uXrDxhbmVxitJ1KDwLS4H3AAwFpaK5h1cQzw,8531
 fusion_bench/utils/strenum/_name_mangler.py,sha256=o11M5-bURW2RBvRTYXFQIPNeqLzburdoWLIqk8X3ydw,3397
 fusion_bench/utils/strenum/_version.py,sha256=6JQRo9LcvODbCOeVFYQb9HNJ_J9XiG_Zbn8ws2A3BV8,18466
-fusion_bench-0.2.25.dist-info/licenses/LICENSE,sha256=nhnOJlw4CPuPVE0qvkGmxfFgHmKi-6nzXvTu8t0NUdg,1066
+fusion_bench-0.2.26.dist-info/licenses/LICENSE,sha256=nhnOJlw4CPuPVE0qvkGmxfFgHmKi-6nzXvTu8t0NUdg,1066
 fusion_bench_config/README.md,sha256=Lc8YSBJ5oxf9KV5kKDivJ9LRyGuraGQPmBbgbdVA-j4,703
 fusion_bench_config/clip-vit-base-patch32_robustness_corrupted.yaml,sha256=7IxLQoLRz-sRWyV8Vqc5kQcmYE_9YQz2_77pmvAkum8,1207
 fusion_bench_config/fabric_model_fusion.yaml,sha256=U8BxsaOvsg9bsEZcIpBE-feo9n9G7Y1kQDHqPVxUYAg,2601
@@ -599,14 +603,14 @@ fusion_bench_config/hydra/default.yaml,sha256=Fpi3pV1hqPoPk5QdBncse6NlNOAl2YHzD4
 fusion_bench_config/hydra/help/fusion_bench_help.yaml,sha256=v8s891Cr5wyxBXGDn_VBBwwRmb0JXOL874Sl-zNoCWA,1880
 fusion_bench_config/hydra/job_logging/rich_logging.yaml,sha256=_dYGeFTCqaPrRowLXBNMXwzYhw8ns1TkQFfALwK1aCw,441
 fusion_bench_config/method/clip_finetune.yaml,sha256=yWjcdKYaKvy53sGaygg2ElAjb9-YFCyCGE1s9aB_dPM,677
-fusion_bench_config/method/depth_upscaling.yaml,sha256=m2XUK8Znf8nnaPKMNH7Un19DQXJlFwpaHE02UId1nxY,632
-fusion_bench_config/method/dummy.yaml,sha256=5qs6OuIfriKOH7FgqvcMXMUoRLmXDZmjA4irpAsc5xo,45
+fusion_bench_config/method/depth_upscaling.yaml,sha256=86YqczaMzZftymLy_k2cb-GMy4C42yTxxP4c4htZTBs,1230
+fusion_bench_config/method/dummy.yaml,sha256=Pw2w6WQiw3K4_KH0THPs4NSM7lZoZLsNbB72iPSVsl8,427
 fusion_bench_config/method/mixtral_moe_merging.yaml,sha256=AdVhXD6Crw-B3QyNpP4ToHRSg-EeSCIGtazA7lQvPOU,148
 fusion_bench_config/method/mixtral_moe_upscaling.yaml,sha256=wYDRnWOpZ6SgvL2Fm9wIDomrN2x5Jaq5vg1hjh3druk,210
-fusion_bench_config/method/model_recombination.yaml,sha256=RGb4boklWcN_GnI-bB5o_qr5o5vGgfIfi_EnTqQ_qcA,195
-fusion_bench_config/method/simple_average.yaml,sha256=GtMNvt0-qWOevRX2V6fjiYUO2BwDvMw-EcxRMS_PhZQ,53
-fusion_bench_config/method/task_arithmetic.yaml,sha256=hqkbc8kbzEFPFmBIKbf-6-vT2ZsBYxhhlP7ZmNT13PM,74
-fusion_bench_config/method/ties_merging.yaml,sha256=0lsy-q-9SNY5xzfoAOFpeva2AqdwcbLwMxb0ZtTU2PA,292
+fusion_bench_config/method/model_recombination.yaml,sha256=DeyVPdDCL-eyJDlPZXLAIWfKi3p8nN0dLFRx5ydsERc,740
+fusion_bench_config/method/simple_average.yaml,sha256=uB51mNlFKb9S3Go1p6SLGgr3PWJFZs97Ccn1zZZkEug,577
+fusion_bench_config/method/task_arithmetic.yaml,sha256=zQmNpnQrZTHiRv_KmYnHPMScKf8MUMLbQYh9254_1Jg,580
+fusion_bench_config/method/ties_merging.yaml,sha256=c3BjnFo-ZU5hmCrfi-1VQPhd_EYGtftxxYDHTVCMy6s,501
 fusion_bench_config/method/ada_svd/clip_vision.yaml,sha256=3l0VKCL66rZNx020UKhf_UzXScZ5XZYOUeNm8mqo0So,183
 fusion_bench_config/method/adamerging/clip.yaml,sha256=NBJaK0a4RxV3D2LRciUeWmTqabRwu6OxZnT7u7iz6ug,753
 fusion_bench_config/method/adamerging/layer_wise_flan_t5.yaml,sha256=DxkZhcuu_-ErIUqBUmWKN5UXYYWKoKPX6IgjV-Txwv0,541
@@ -614,7 +618,7 @@ fusion_bench_config/method/adamerging/layer_wise_gpt2.yaml,sha256=bLz6zc5CofeUO2
 fusion_bench_config/method/adamerging/llama_sft.yaml,sha256=khKzfhvQ5oxBMH0d-YvyjN-qIgQNeevDodXngS5g9KY,1022
 fusion_bench_config/method/analysis/task_vector_cos_similarity.yaml,sha256=hxVA4deUr1go1RZl12qD8PekwydWJ9SBQowSqmo3A8I,139
 fusion_bench_config/method/analysis/task_vector_violin_plot.yaml,sha256=FmBGj0Ib2xYd-49x_xZSeVbExwL-A9-tHhHTMBrT_Fg,134
-fusion_bench_config/method/bitdelta/bitdelta.yaml,sha256=b92xQpufqrSHAiU0QFE8g0nQ7RGSowOubGrEz_KugsQ,231
+fusion_bench_config/method/bitdelta/bitdelta.yaml,sha256=uuR5x1IVTWyZjTSd5i1JXd_D8tG7tWBfOpgMBDCBgR0,436
 fusion_bench_config/method/classification/clip_continual_finetune.yaml,sha256=Ls63kdLb1bLwUEqzfyTtJcpFOdv3HmwzBML0V2JnnAs,791
 fusion_bench_config/method/classification/clip_finetune.yaml,sha256=yWjcdKYaKvy53sGaygg2ElAjb9-YFCyCGE1s9aB_dPM,677
 fusion_bench_config/method/classification/image_classification_finetune.yaml,sha256=fl60RFCYwmrwwu3QlaJTFiBLmSmnjHxl-xyq4Gb80iU,401
@@ -631,9 +635,10 @@ fusion_bench_config/method/dare/task_arithmetic.yaml,sha256=cUAweNJ6p2aOv__0dvUL
 fusion_bench_config/method/dare/ties_merging.yaml,sha256=7gDW4XpezrsccsbJGqqKrbX26JnqAc85A-MY66DGvuE,416
 fusion_bench_config/method/dawe/dawe_for_clip.yaml,sha256=99P5xpp1YGvIwXGxDcxRtJMLE2FhvEFmFBQjOMEcGoc,1023
 fusion_bench_config/method/doge_ta/doge_ta.yaml,sha256=CtZI3YPMJNDy225yhOJbSiMKlsc-X5nCFzmVh0dvr-w,78
-fusion_bench_config/method/ensemble/max_model_predictor.yaml,sha256=khdpCvKMNytx4nZSgtUJFXv44MVytXu0aqUVd9TixXo,57
-fusion_bench_config/method/ensemble/simple_ensemble.yaml,sha256=RKa3IgN3DfFZVmeXVIdTt0NdPVV0jFkpQz6SxLs3Kso,124
-fusion_bench_config/method/ensemble/weighted_ensemble.yaml,sha256=2KD3PjFglqL7fjqhjXtOWxZ1mvmYodiNVroXsFd7EGE,261
+fusion_bench_config/method/dop/dop.yaml,sha256=ZgdjuVfTj83kAvrS4RrPgGX7d_QQ7d1lIMlzhjiVeUc,954
+fusion_bench_config/method/ensemble/max_model_predictor.yaml,sha256=ugO9FbEYqQk3RkX7wUDE9UOg-4D0F4Rezv0O-7hTeRg,476
+fusion_bench_config/method/ensemble/simple_ensemble.yaml,sha256=kfPAaPVQIet9dYThKNsEBfe9gHdeCREnsM-snSOPahM,546
+fusion_bench_config/method/ensemble/weighted_ensemble.yaml,sha256=LhlxU2P_inxR8MB0Z62phHWj5S4qxD7ITG4Ly-GUcQo,770
 fusion_bench_config/method/expert_sparsity/README.md,sha256=CLE0-XblXDWCUTHPaTNtBH-YquXn-uawwTJiYrgjMaA,239
 fusion_bench_config/method/expert_sparsity/mixtral.yaml,sha256=maFL3LM0zfnQ1eXoNXUslSjgZmpOdUJgl_a31dYUBbc,605
 fusion_bench_config/method/fisher_merging/clip_fisher_merging.yaml,sha256=-m5uDA9hfBg_8vF3s0MnUp0JTl3MqpB4-rlPEg9CHD4,569
@@ -646,23 +651,23 @@ fusion_bench_config/method/gossip/layer_wise_flan_t5.yaml,sha256=2yBqbhwz2vq65wT
 fusion_bench_config/method/isotropic_merging/iso_c.yaml,sha256=mn_5nyc7s_a7QH1MkEj9ZncjNHtZa0mzfXcUGRJOiAw,81
 fusion_bench_config/method/isotropic_merging/iso_cts.yaml,sha256=70BODJt69pZ_9xH7S_Z2Yzb299LFIGkXy1bQiHQad6A,110
 fusion_bench_config/method/linear/expo.yaml,sha256=St3NW6cKVRV3vCn8y0gxQ8k66VTdtsLTEWQTbO9wQ0Y,420
-fusion_bench_config/method/linear/linear_interpolation.yaml,sha256=chM6_HRKKcMleTeuKY3-YNI1qaMG2CfnsRwUxAlHsRw,66
+fusion_bench_config/method/linear/linear_interpolation.yaml,sha256=cAL_ekEIJhJD4cfAbKilV0k_lNNPoJqY4sABVEKcM7E,523
 fusion_bench_config/method/linear/llama_expo.yaml,sha256=SvqamjT06BMObQ58sks5x7Wv6kGpp3-Nlw3ihbD_kSA,621
 fusion_bench_config/method/linear/llama_expo_with_dare.yaml,sha256=Pp8s2xmEg5XSvaGKtwTYx_PzcGvwRh2gPpZ6u9as4_E,383
 fusion_bench_config/method/linear/simple_average_for_causallm.yaml,sha256=qqeIr61PJEcfZclZ5vV64GCzyt-8b1zB0FDZu8DsbXQ,322
 fusion_bench_config/method/linear/task_arithmetic_for_causallm.yaml,sha256=tJA0n0_XVvll4rZYVHQVqFCz8W3Bey6NjPKMIH3-P0U,142
 fusion_bench_config/method/linear/ties_merging_for_causallm.yaml,sha256=1oEIdxV0OqWjDQ9V_lmXEPUayp4KbKHE2SvpCLmiKOU,489
-fusion_bench_config/method/linear/weighted_average.yaml,sha256=uq2gHGCwVHHSa1H-hzcrSlumUTLJ50tfyiY1Mh1pFsk,186
-fusion_bench_config/method/linear/weighted_average_for_llama.yaml,sha256=se2aq6t5R1f-ZG6ubUyRr__DBe9BzXrgL81ua3DkQoM,498
+fusion_bench_config/method/linear/weighted_average.yaml,sha256=OjE4EdfDHPYx8PlBJ6xIpCz4ITu_65VsRyefioRXGQ8,408
+fusion_bench_config/method/linear/weighted_average_for_llama.yaml,sha256=886ZKr81gyN7DISqtbrM5WnjSXd_6AlakQyOJQagoYY,518
 fusion_bench_config/method/lm_finetune/bradley_terry_rm.yaml,sha256=QHsRfJK9K4KajsX3LBHG8cDt7ZLJWxOBnJjpHRQSB_s,1348
 fusion_bench_config/method/lm_finetune/fullfinetune_sft.yaml,sha256=c0rFqj2GV11X9RMraHXJtJ9OiMUzZtvDVsTn4tgAeco,1337
 fusion_bench_config/method/lm_finetune/peftfinetune_sft.yaml,sha256=LjGwfTiiC5iQKr62i22XopQTfSKbx9UbsDvEW-byneQ,1622
-fusion_bench_config/method/model_stock/model_stock.yaml,sha256=G92eRhG_Zsgi2R2FRnMViGC9QPvo7ge-o_eI4ZZLxao,321
+fusion_bench_config/method/model_stock/model_stock.yaml,sha256=4KHAFCjL4AQ5dxkv7IGkUTxE8g-GCoxDkA3BbnlzQC0,530
 fusion_bench_config/method/moe_pruner/moe_pruner.yaml,sha256=OYMYLKvLlNEht7BK9phaTEvAE1ySaVi-pvjYiT-OTGw,442
-fusion_bench_config/method/opcm/opcm.yaml,sha256=YkjAMVGFDj0xqqxA7XWNr0vmcRyxeYbV387nWe0cUbk,331
-fusion_bench_config/method/opcm/task_arithmetic.yaml,sha256=wc9Bz7K_u0feLZbhCBhAuwjeIQTSugJu0I0DCmRNY_c,326
-fusion_bench_config/method/opcm/ties_merging.yaml,sha256=XOE1XzSdYXYzqev9bFD4g4prcmE1OiVINkVXsquizAA,541
-fusion_bench_config/method/opcm/weight_average.yaml,sha256=SmhftSJ_YXN6tn-0GuzQgjbE2sOd7YXoPYjDWzpY_9E,304
+fusion_bench_config/method/opcm/opcm.yaml,sha256=7NBOGo6W1FDbqdkT8gfM5PI2kHfqB8ofMfgcxVI1suM,686
+fusion_bench_config/method/opcm/task_arithmetic.yaml,sha256=WL_nVXhZWV9fe_ttChShkjYZVJnOCzvZ3i7NBppYsxk,743
+fusion_bench_config/method/opcm/ties_merging.yaml,sha256=1-xR0dVEEFJue9r-oBk1ZfGmGM9vCu4cJBG5aZnJ3C8,917
+fusion_bench_config/method/opcm/weight_average.yaml,sha256=n-eyxVkpRanlRJdFWFK3kppiO_W1S99WNjyjdBLDnw0,668
 fusion_bench_config/method/pruning/llama_magnitude_pruning.yaml,sha256=Px8LU_UtDz-YHDFfqQ7scEPOproiFOaudKVshrhCTgc,483
 fusion_bench_config/method/pruning/llama_random_pruning.yaml,sha256=0RiZS8d42PXZzwncPG8zcbnyYJ9vtfr2sOSqS8oDyT4,325
 fusion_bench_config/method/pruning/llama_sparsegpt_pruning.yaml,sha256=gC6Ss0n2tKSb4gyVfx45BvsFbVBGN-om4-2S1sKS-_w,505
@@ -679,15 +684,15 @@ fusion_bench_config/method/regmean/clip_regmean.yaml,sha256=QfkCHCLK9wbyB1Tq1S7Y
 fusion_bench_config/method/regmean/gpt2_regmean.yaml,sha256=n94aTboDdwSA7Tki8l_o8tYQkhXxPV8lRf-dRNPIsOs,422
 fusion_bench_config/method/regmean/regmean.yaml,sha256=ZgVVLx-lHwVgjtjTl4VZUlthh8yyua87QvoJfmNHud4,101
 fusion_bench_config/method/regmean_plusplus/clip_regmean_plusplus.yaml,sha256=A034ryEwvosqyQzA3KWs7kdp-3CUnoJtCujVywV-uzA,434
-fusion_bench_config/method/slerp/slerp.yaml,sha256=xldDUULtfCdwzAkQUb0C8-TmbW7FqcAlIOsPX8p4n6w,116
-fusion_bench_config/method/slerp/slerp_lm.yaml,sha256=c5OQ0zD7e0lXQyec09joHOFNxV1LMT4bHuwgk9GWskc,114
+fusion_bench_config/method/slerp/slerp.yaml,sha256=XR3z6iqyHirkoFSdLAeV2bP1yyI25MoWG-LqdE-ypjA,719
+fusion_bench_config/method/slerp/slerp_lm.yaml,sha256=hO07n6elZg_FrqEfSfbdR-tb1hqwT7vaLgAZKdF8O1o,479
 fusion_bench_config/method/smile_upscaling/causal_lm_upscaling.yaml,sha256=skLwgu_VHShm4m0oEOkqKzcBS5Cz7J29xEj7pTaSm0k,916
 fusion_bench_config/method/smile_upscaling/error_accumulation.yaml,sha256=6Gui-OuQ3P_4TwO_syh9SWJCNeHiAQzS55aO-ByYKbQ,154
 fusion_bench_config/method/smile_upscaling/projected_energy.yaml,sha256=M_EBOC3B_pxaBO3tD6mnbXpvy6-EaegSsE-jdJs-HY0,114
 fusion_bench_config/method/smile_upscaling/singular_projection_merging.yaml,sha256=ZMn_ImRjjc2uozf7ocQIzbgvFDpBV7S-34KptbBXVGo,200
 fusion_bench_config/method/smile_upscaling/smile_mistral_upscaling.yaml,sha256=VFMrkbO69d0wCjTQCuKysYGVe6hEwNu792g1QkhU5Mk,383
 fusion_bench_config/method/smile_upscaling/smile_qwen2_upscaling.yaml,sha256=MfZ1u1HIJoy_csWiLzR4GLz-eiaVxo2gmNYre224yqo,433
-fusion_bench_config/method/smile_upscaling/smile_upscaling.yaml,sha256=G88mabTTniDUtiUC9Vg3cj_sw6D05mE4_ZdyYI4Omjk,477
+fusion_bench_config/method/smile_upscaling/smile_upscaling.yaml,sha256=38DGdOjpDo-dOMpfy807p3x-eAvibjED-BGtFGnaycA,689
 fusion_bench_config/method/sparselo_pruning/llama_iterative_sparselo.yaml,sha256=L-WgNhFjcp_2tocDxZi6STVTtoaSd1v9UOQaKO_QvHM,669
 fusion_bench_config/method/sparselo_pruning/llama_pcp_sparselo.yaml,sha256=prTEFH0eu7R_CVNQ0GPWL9QsOLFcT1uM12zZdi3qcFo,636
 fusion_bench_config/method/sparselo_pruning/llama_sparselo.yaml,sha256=Cmg8N4l--3C0qeSHG-HLOgjJZ954eWHoDNgRnx0pLK0,614
@@ -948,8 +953,8 @@ fusion_bench_config/taskpool/LMEvalHarnessTaskPool/lm_eval.yaml,sha256=3q-KMuFaM
 fusion_bench_config/taskpool/OpenCLIPVisionModelTaskPool/ViT-B-16_TA8.yaml,sha256=GjpiiRownrBCpl-TNwWRW2PYePbF-Cl99jlLNPrK5T4,1017
 fusion_bench_config/taskpool/OpenCLIPVisionModelTaskPool/ViT-B-32_TA8.yaml,sha256=WwiYMQKehtJixDPnu5o3vcWe4yJksXTWRqOzm3uVWXQ,1017
 fusion_bench_config/taskpool/OpenCLIPVisionModelTaskPool/ViT-L-14_TA8.yaml,sha256=xGRt0J9joXTzWUew6DvoYprAWlPXhaVFw5AX4im5VQw,1017
-fusion_bench-0.2.25.dist-info/METADATA,sha256=hOFNvf8-PM-SP8-58zf4yeOFX27dLWS27Ow1PaPpu30,22621
-fusion_bench-0.2.25.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-fusion_bench-0.2.25.dist-info/entry_points.txt,sha256=iUQ8MCJvda7HP4vYh2n1Teoapb4G9PBVYZkAfcc5SHU,116
-fusion_bench-0.2.25.dist-info/top_level.txt,sha256=BuO4TL6iHL_2yPBUX9-LlIrHRczA_BNMIFwweK0PQEI,13
-fusion_bench-0.2.25.dist-info/RECORD,,
+fusion_bench-0.2.26.dist-info/METADATA,sha256=BOHkLorLs0w_fgAtRz7tpYVExKxFiClGISLlsnW3BG8,24307
+fusion_bench-0.2.26.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fusion_bench-0.2.26.dist-info/entry_points.txt,sha256=iUQ8MCJvda7HP4vYh2n1Teoapb4G9PBVYZkAfcc5SHU,116
+fusion_bench-0.2.26.dist-info/top_level.txt,sha256=BuO4TL6iHL_2yPBUX9-LlIrHRczA_BNMIFwweK0PQEI,13
+fusion_bench-0.2.26.dist-info/RECORD,,

fusion_bench_config/method/bitdelta/bitdelta.yaml

@@ -1,3 +1,6 @@
+# =============================================================================
+# FusionBench Method Configuration: BitDelta
+# =============================================================================
 _target_: fusion_bench.method.bitdelta.BitDeltaAlgorithm
 save_dir: null
 save_full_model: false

fusion_bench_config/method/depth_upscaling.yaml

@@ -1,3 +1,12 @@
+# =============================================================================
+# FusionBench Method Configuration: Depth Upscaling
+# =============================================================================
+# Constructs a deeper model by stacking/selecting layers from existing models.
+#
+# - layer_indices: list[int | str] specifying which layers to use. Strings are Python
+#   expressions evaluated to lists, e.g., "range(6,12)".
+# - Example: [0, 2, 4, "range(6,12)"] selects 1st, 3rd, 5th, and 7th-12th layers.
+# =============================================================================
 _target_: DepthUpscalingAlgorithm
 # this should be a list of integers or string, indicating the sequence of layers. 
 # If the entry is an integer, it will use the n-th layer of the model. 

fusion_bench_config/method/dop/dop.yaml

@@ -0,0 +1,30 @@
+_target_: fusion_bench.method.dop.dop.ContinualDOPForCLIP
+
+# the random seed to use
+seed: null
+# shuffle the order of the models
+shuffle_order: true
+# save the merged model on every step
+save_on_every_step: false
+# evaluate the merged model on every step
+evaluate_on_every_step: true
+
+# optimizer (learning rate)
+lr: 1e-4
+# optimizer (num_steps)
+num_steps: 200
+
+# weighted loss
+# if mgda is true, use mgda to optimize the loss weights
+mgda: true
+# if mgda is false, this is the weight for the loss of the first task
+alpha: 0.8
+# if mgda is true and ema is ture, using exponential moving average (ema), alpha is the initial value
+ema: true
+# if mgda is true and ema is ture, using exponential moving average (ema), beta is the decay rate
+ema_beta: 0.999
+
+# epsilon for svd (the proportion of energy retained)
+svd_epsilon: 0.99999
+# the space to project the delta w (left singular vectors, right singular vectors, or both)
+svd_proj_space: uv # u or v or uv

fusion_bench_config/method/dummy.yaml

@@ -1 +1,7 @@
+# =============================================================================
+# FusionBench Method Configuration: Dummy
+# =============================================================================
+# No-op method for testing pipelines and wiring.
+# Instantiates and exits without modifying models.
+# =============================================================================
 _target_: fusion_bench.method.DummyAlgorithm

fusion_bench_config/method/ensemble/max_model_predictor.yaml

@@ -1 +1,7 @@
+# =============================================================================
+# FusionBench Method Configuration: Max Model Predictor
+# =============================================================================
+# Selects the model with maximum confidence or performance per example/task.
+# No additional hyperparameters are required.
+# =============================================================================
 _target_: fusion_bench.method.MaxModelPredictorAlgorithm

fusion_bench_config/method/ensemble/simple_ensemble.yaml

@@ -1,2 +1,9 @@
+# =============================================================================
+# FusionBench Method Configuration: Simple Ensemble
+# =============================================================================
+# Averages model predictions uniformly.
+#
+# device_map: leave null for single device or provide a mapping for multi-device setups.
+# =============================================================================
 _target_: fusion_bench.method.SimpleEnsembleAlgorithm
-device_map: null  # Set to null for single device, or specify mapping
+device_map: null # Set to null for single device, or specify mapping

fusion_bench_config/method/ensemble/weighted_ensemble.yaml

@@ -1,3 +1,11 @@
+# =============================================================================
+# FusionBench Method Configuration: Weighted Ensemble
+# =============================================================================
+# Ensembles model predictions using specified per-model weights.
+#
+# - Set normalize=true to rescale weights to sum to 1.
+# - weights: one float per model in the pool (order-sensitive). If null, uses equal weights.
+# =============================================================================
 _target_: fusion_bench.method.WeightedEnsembleAlgorithm
 normalize: true
 # this should be a list of floats, one for each model in the ensemble

fusion_bench_config/method/linear/linear_interpolation.yaml

@@ -1,2 +1,10 @@
+# =============================================================================
+# FusionBench Method Configuration: Linear Interpolation
+# =============================================================================
+# Interpolates between two models: (1 - t) * model0 + t * model1
+#
+# - t in [0,1]: 0 returns model0; 1 returns model1.
+# - Only meaningful for two-model pools.
+# =============================================================================
 _target_: fusion_bench.method.LinearInterpolationAlgorithm
 t: 0.5

fusion_bench_config/method/linear/weighted_average.yaml

@@ -1,3 +1,6 @@
+# =============================================================================
+# FusionBench Method Configuration: Weighted Average (Linear)
+# =============================================================================
 _target_: fusion_bench.method.WeightedAverageAlgorithm
 normalize: true # if true, the weights will be normalized before merging
 weights: # List of weights for each model

fusion_bench_config/method/linear/weighted_average_for_llama.yaml

@@ -1,4 +1,4 @@
-_target_: WeightedAverageForLLama
+_target_: fusion_bench.method.WeightedAverageForLLama
 normalize: true # if true, the weights will be normalized before merging
 weights: # List of weights for each model
   - 0.5

fusion_bench_config/method/model_recombination.yaml

@@ -1,3 +1,11 @@
+# =============================================================================
+# FusionBench Method Configuration: Model Recombination
+# =============================================================================
+# Recombines submodules/layers from multiple models to form a new model.
+#
+# - return_modelpool: override run() argument to return model pool instead of merged model.
+#   Set to null to respect runtime argument; set to true/false to force behavior.
+# =============================================================================
 _target_: fusion_bench.method.ModelRecombinationAlgorithm
 # if `return_model_pool` is not null, the argument `return_modelpool` passed to the `run` method will be ignored.
 return_modelpool: null

fusion_bench_config/method/model_stock/model_stock.yaml

@@ -1,3 +1,6 @@
+# =============================================================================
+# FusionBench Method Configuration: Model Stock
+# =============================================================================
 _target_: fusion_bench.method.model_stock.ModelStock
 ignore_keys:
   [
@@ -9,4 +12,4 @@ ignore_keys:
     "model.ln_final.bias",
   ]
 model_save_path: ${path.log_dir}/checkpoint
-model_save_kwargs: null
+model_save_kwargs: null

fusion_bench_config/method/opcm/opcm.yaml

@@ -1,3 +1,8 @@
+# =============================================================================
+# FusionBench Method Configuration: OPCM
+# =============================================================================
+# Incrementally merges models via SVD projection and evaluation per step.
+# =============================================================================
 _target_: fusion_bench.method.opcm.opcm.OPCMForCLIP
 # shuffle the order of the models
 shuffle_order: true

fusion_bench_config/method/opcm/task_arithmetic.yaml

@@ -1,3 +1,9 @@
+# =============================================================================
+# FusionBench Method Configuration: Continual Task Arithmetic
+# =============================================================================
+# Applies task arithmetic incrementally across a stream of models.
+# Maintains per-step save/eval similar to OPCM.
+# =============================================================================
 _target_: fusion_bench.method.opcm.task_arithmetic.ContinualTaskArithmeticForCLIP
 scaling_factor: 0.3
 # shuffle the order of the models

fusion_bench_config/method/opcm/ties_merging.yaml

@@ -1,3 +1,8 @@
+# =============================================================================
+# FusionBench Method Configuration: Continual TIES Merging
+# =============================================================================
+# Continual variant of TIES merging with per-step save/eval instrumentation.
+# =============================================================================
 _target_: fusion_bench.method.opcm.ties_merging.ContinualTiesMergingForCLIP
 # Scaling factor $\lambda$
 scaling_factor: 0.5

fusion_bench_config/method/opcm/weight_average.yaml

@@ -1,3 +1,8 @@
+# =============================================================================
+# FusionBench Method Configuration: Continual Weighted Average
+# =============================================================================
+# Incrementally averages model weights as new models arrive.
+# =============================================================================
 _target_: fusion_bench.method.opcm.weight_average.ContinualWeightAverageForCLIP
 # shuffle the order of the models
 shuffle_order: true

fusion_bench_config/method/simple_average.yaml

@@ -1 +1,10 @@
+# =============================================================================
+# FusionBench Method Configuration: Simple Average
+# =============================================================================
+# Equally averages parameters of all models in the model pool.
+#
+# Usage notes
+# - No hyperparameters required; behavior is deterministic given model order.
+# - Ensure models are architecture-compatible (same shapes) before merging.
+# =============================================================================
 _target_: fusion_bench.method.SimpleAverageAlgorithm

fusion_bench_config/method/slerp/slerp.yaml

@@ -1,3 +1,12 @@
+# =============================================================================
+# FusionBench Method Configuration: Spherical Linear Interpolation (SLERP)
+# =============================================================================
+# Interpolates between two parameter vectors on a hypersphere.
+#
+# - t in [0,1]: interpolation factor; 0 returns model0; 1 returns model1.
+# - DOT_THRESHOLD: threshold to switch to linear interpolation when vectors are near-aligned.
+# - epsilon: small constant to avoid division by zero.
+# =============================================================================
 _target_: fusion_bench.method.SlerpMergeAlgorithm
 t: 0.5 # interpolation factor
 DOT_THRESHOLD: 0.9995

fusion_bench_config/method/slerp/slerp_lm.yaml

@@ -1,3 +1,8 @@
+# =============================================================================
+# FusionBench Method Configuration: SLERP for Causal LM
+# =============================================================================
+# Spherical linear interpolation between two causal language models.
+# =============================================================================
 _target_: fusion_bench.method.SlerpForCausalLM
 t: 0.5
 model_save_path: ${path.log_dir}/checkpoint

fusion_bench_config/method/smile_upscaling/smile_upscaling.yaml

@@ -1,3 +1,6 @@
+# =============================================================================
+# FusionBench Method Configuration: SMILE Upscaling
+# =============================================================================
 _target_: fusion_bench.method.SmileUpscalingAlgorithm
 # merge device on cuda can accelerate the SVD computation
 device: cpu

fusion_bench_config/method/task_arithmetic.yaml

@@ -1,2 +1,11 @@
+# =============================================================================
+# FusionBench Method Configuration: Task Arithmetic
+# =============================================================================
+# Performs task vector arithmetic: base + lambda * \sum_i (task_i - base).
+#
+# Notes
+# - scaling_factor controls the contribution of the task delta.
+# - Model compatibility is required (matching parameter shapes).
+# =============================================================================
 _target_: fusion_bench.method.TaskArithmeticAlgorithm
 scaling_factor: 0.3

fusion_bench_config/method/ties_merging.yaml

@@ -1,3 +1,6 @@
+# =============================================================================
+# FusionBench Method Configuration: Ties Merging
+# =============================================================================
 _target_: fusion_bench.method.TiesMergingAlgorithm
 # Scaling factor $\lambda$
 scaling_factor: 0.3