fusion-bench 0.2.29__tar.gz → 0.2.30__tar.gz

{fusion_bench-0.2.29 → fusion_bench-0.2.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fusion-bench
-Version: 0.2.29
+Version: 0.2.30
 Summary: A Comprehensive Benchmark of Deep Model Fusion
 Author-email: Anke Tang <tang.anke@foxmail.com>
 Project-URL: Repository, https://github.com/tanganke/fusion_bench
@@ -255,9 +255,9 @@ First, create a new Python file for the algorithm in the `fusion_bench/method` d
 Following the naming convention, the file should be named `{method_name_or_class}/{variant}.py`.
 
 ```python
-from fusion_bench import BaseModelFusionAlgorithm, BaseModelPool
+from fusion_bench import BaseAlgorithm, BaseModelPool
 
-class DerivedModelFusionAlgorithm(BaseModelFusionAlgorithm):
+class DerivedModelFusionAlgorithm(BaseAlgorithm):
     """
     An example of a derived model fusion algorithm.
     """
@@ -265,7 +265,7 @@ class DerivedModelFusionAlgorithm(BaseModelFusionAlgorithm):
     # _config_mapping maps the attribution to the corresponding key in the configuration file.
     # this is optional and can be used to serialize the object to a configuration file.
     # `self.config.hyperparam_1` will be mapped to the attribute `hyperparam_attr_1`.
-    _config_mapping = BaseModelFusionAlgorithm._config_mapping | {
+    _config_mapping = BaseAlgorithm._config_mapping | {
         "hyperparam_attr_1": "hyperparam_1",
         "hyperparam_attr_2": "hyperparam_2",
     }
@@ -344,9 +344,9 @@ If you find this benchmark useful, please consider citing our work:
 ```bibtex
 @article{tang2024fusionbench,
   title={Fusionbench: A comprehensive benchmark of deep model fusion},
-  author={Tang, Anke and Shen, Li and Luo, Yong and Hu, Han and Du, Bo and Tao, Dacheng},
-  journal={arXiv preprint arXiv:2406.03280},
-  year={2024}
+  author={Tang, Anke and Shen, Li and Luo, Yong and Yang, Enneng and Hu, Han and Zhang, Lefei and Du, Bo and Tao, Dacheng},
+  journal={Journal of Machine Learning Research},
+  year={2025}
 }
 ```
 

{fusion_bench-0.2.29 → fusion_bench-0.2.30}/README.md

@@ -215,9 +215,9 @@ First, create a new Python file for the algorithm in the `fusion_bench/method` d
 Following the naming convention, the file should be named `{method_name_or_class}/{variant}.py`.
 
 ```python
-from fusion_bench import BaseModelFusionAlgorithm, BaseModelPool
+from fusion_bench import BaseAlgorithm, BaseModelPool
 
-class DerivedModelFusionAlgorithm(BaseModelFusionAlgorithm):
+class DerivedModelFusionAlgorithm(BaseAlgorithm):
     """
     An example of a derived model fusion algorithm.
     """
@@ -225,7 +225,7 @@ class DerivedModelFusionAlgorithm(BaseModelFusionAlgorithm):
     # _config_mapping maps the attribution to the corresponding key in the configuration file.
     # this is optional and can be used to serialize the object to a configuration file.
     # `self.config.hyperparam_1` will be mapped to the attribute `hyperparam_attr_1`.
-    _config_mapping = BaseModelFusionAlgorithm._config_mapping | {
+    _config_mapping = BaseAlgorithm._config_mapping | {
         "hyperparam_attr_1": "hyperparam_1",
         "hyperparam_attr_2": "hyperparam_2",
     }
@@ -304,9 +304,9 @@ If you find this benchmark useful, please consider citing our work:
 ```bibtex
 @article{tang2024fusionbench,
   title={Fusionbench: A comprehensive benchmark of deep model fusion},
-  author={Tang, Anke and Shen, Li and Luo, Yong and Hu, Han and Du, Bo and Tao, Dacheng},
-  journal={arXiv preprint arXiv:2406.03280},
-  year={2024}
+  author={Tang, Anke and Shen, Li and Luo, Yong and Yang, Enneng and Hu, Han and Zhang, Lefei and Du, Bo and Tao, Dacheng},
+  journal={Journal of Machine Learning Research},
+  year={2025}
 }
 ```
 

{fusion_bench-0.2.29 → fusion_bench-0.2.30}/fusion_bench/method/__init__.py

@@ -144,7 +144,15 @@ _extra_objects = {
 
 if TYPE_CHECKING:
     from .ada_svd import AdaSVDMergingForCLIPVisionModel
-    from .adamerging import *
+    from .adamerging import (
+        CLIPLayerWiseAdaMergingAlgorithm,
+        CLIPTaskWiseAdaMergingAlgorithm,
+        FlanT5LayerWiseAdaMergingAlgorithm,
+        GPT2LayerWiseAdaMergingAlgorithm,
+        LayerWiseAdaMergingForLlamaSFT,
+        ResNetLayerWiseAdamerging,
+        ResNetTaskWiseAdamerging,
+    )
     from .analysis import TaskVectorCosSimilarity, TaskVectorViolinPlot
     from .base_algorithm import BaseAlgorithm, BaseModelFusionAlgorithm
     from .bitdelta import BitDeltaAlgorithm

{fusion_bench-0.2.29 → fusion_bench-0.2.30}/fusion_bench/method/base_algorithm.py

@@ -40,6 +40,7 @@ from typing import Optional  # noqa: F401
 
 from fusion_bench.mixins import BaseYAMLSerializable
 from fusion_bench.modelpool import BaseModelPool
+from fusion_bench.utils.misc import DeprecationWarningMeta
 
 __all__ = ["BaseAlgorithm", "BaseModelFusionAlgorithm"]
 
@@ -202,27 +203,36 @@ class BaseAlgorithm(BaseYAMLSerializable):
         pass
 
 
-BaseModelFusionAlgorithm = BaseAlgorithm
-"""
-Alias for BaseAlgorithm class.
+# Create a deprecated wrapper class that inherits from BaseAlgorithm
+class BaseModelFusionAlgorithm(BaseAlgorithm, metaclass=DeprecationWarningMeta):
+    """
+    Alias for BaseAlgorithm class.
 
-This alias is provided for backward compatibility and semantic clarity.
-Some users may prefer the more explicit name 'BaseModelFusionAlgorithm'
-to emphasize that this class is specifically designed for model fusion
-tasks, while others may prefer the shorter 'BaseAlgorithm' name.
+    .. deprecated::
+        BaseModelFusionAlgorithm is deprecated and will be removed in a future version.
+        Use :class:`BaseAlgorithm` instead.
 
-Both names refer to the exact same class and can be used interchangeably.
+    This alias was provided for backward compatibility and semantic clarity.
+    Both names refer to the same base class and can be used interchangeably,
+    but BaseAlgorithm is now the preferred name for all implementations.
 
-Examples:
-    Using the original name:
-    >>> class MyAlgorithm(BaseAlgorithm):
-    ...     def run(self, modelpool): pass
+    Examples:
+        Preferred (using BaseAlgorithm):
 
-    Using the alias:
-    >>> class MyAlgorithm(BaseModelFusionAlgorithm):
-    ...     def run(self, modelpool): pass
+        >>> class MyAlgorithm(BaseAlgorithm):
+        ...     def run(self, modelpool): pass
 
-Note:
-    The alias is maintained for compatibility but BaseAlgorithm is the
-    preferred name for new implementations.
-"""
+        Deprecated (using BaseModelFusionAlgorithm):
+
+        >>> class MyAlgorithm(BaseModelFusionAlgorithm):  # Will trigger deprecation warning
+        ...     def run(self, modelpool): pass
+
+    Note:
+        New implementations should use :class:`BaseAlgorithm` exclusively.
+        The BaseModelFusionAlgorithm alias will be removed in a future release.
+
+    Warning:
+        Using BaseModelFusionAlgorithm will trigger a DeprecationWarning.
+    """
+
+    pass

fusion_bench-0.2.30/fusion_bench/metrics/model_kinship/__init__.py

@@ -0,0 +1,2 @@
+# Exploring Model Kinship for Merging LLMs
+# The implementation of this module is borrowed from: https://github.com/zjunlp/ModelKinship/

fusion_bench-0.2.30/fusion_bench/metrics/model_kinship/calculate.py

@@ -0,0 +1,77 @@
+import logging
+from typing import List
+
+import numpy
+import torch
+
+from .utility import Metric
+
+
+def cosine_similarity(a, b):
+    similarity = numpy.sqrt(numpy.dot(a, b) ** 2 / (numpy.dot(a, a) * numpy.dot(b, b)))
+    return similarity
+
+
+def calculate_model_kinship(
+    delta1: numpy.ndarray, delta2: numpy.ndarray, metrics: List[str]
+) -> dict:
+    """
+    Calculate model kinship using specified metrics.
+
+    Args:
+        delta1: Delta parameters for first model
+        delta2: Delta parameters for second model
+        metrics: List of metrics to calculate
+
+    Returns:
+        dict: Dictionary of metric names and their calculated values
+    """
+    results = {}
+    for metric in metrics:
+        try:
+            if metric not in Metric.list():
+                raise ValueError(f"Unsupported metric: {metric}")
+            results[metric] = calculate_metric(delta1, delta2, metric)
+        except Exception as e:
+            results[metric] = f"Error calculating {metric}: {str(e)}"
+    return results
+
+
+def calculate_metric(
+    d_vector_1: torch.Tensor, d_vector_2: torch.Tensor, metric: str
+) -> str:
+    """
+    Calculate the specified metric between two delta vectors.
+
+    Args:
+        d_vector_1 (torch.Tensor): Delta parameters for model 1.
+        d_vector_2 (torch.Tensor): Delta parameters for model 2.
+        metric (str): The metric to calculate ('pcc', 'ed', 'cs').
+
+    Returns:
+        str: A formatted string with the result of the chosen metric.
+    """
+    logging.info(f"Starting calculation of {metric.upper()} metric...")
+
+    # Pearson Correlation Coefficient (PCC)
+    if metric == "pcc":
+        # Stack the two vectors and calculate the Pearson correlation coefficient
+        stack = torch.stack((d_vector_1, d_vector_2), dim=0)
+        pcc = torch.corrcoef(stack)[0, 1].item()
+        return f"Model Kinship based on Pearson Correlation Coefficient: {pcc}"
+
+    # Euclidean Distance (ED)
+    elif metric == "ed":
+        # Compute the Euclidean distance between the vectors
+        distance = torch.dist(d_vector_1, d_vector_2).item()
+        return f"Model Kinship based on Euclidean Distance: {distance}"
+
+    # Cosine Similarity (CS)
+    elif metric == "cs":
+        # Compute cosine similarity
+        cs = cosine_similarity(d_vector_1, d_vector_2)
+        return f"Model Kinship based on Cosine Similarity: {cs}"
+
+    # If metric is not recognized
+    else:
+        return "Invalid metric specified."

fusion_bench-0.2.30/fusion_bench/metrics/model_kinship/calculate_split.py

@@ -0,0 +1,171 @@
+import logging
+from typing import Dict, List
+
+import numpy
+import torch
+from tqdm import tqdm
+
+from .utility import Metric, load_model_state_dict, quantize_8bit
+
+
+def cosine_similarity(a, b):
+    similarity = numpy.sqrt(numpy.dot(a, b) ** 2 / (numpy.dot(a, a) * numpy.dot(b, b)))
+    return similarity
+
+
+def calculate_model_kinship_split(
+    model_1_name: str,
+    model_2_name: str,
+    model_base_name: str,
+    low_precision: bool,
+    metrics: List[str],
+    device: str = "cuda" if torch.cuda.is_available() else "cpu",
+) -> dict:
+
+    # Extract state dictionaries from models
+    state_dict_1 = load_model_state_dict(model_1_name, device)
+    state_dict_2 = load_model_state_dict(model_2_name, device)
+    state_dict_base = load_model_state_dict(model_base_name, device)
+    results = {}
+
+    # Validate metrics before processing
+    valid_metrics = Metric.list()
+    for metric in metrics:
+        try:
+            if metric not in valid_metrics:
+                raise ValueError(
+                    f"Unsupported metric: {metric}. Valid metrics are: {', '.join(valid_metrics)}"
+                )
+            results[metric] = calculate_metrics_by_split(
+                state_dict_1, state_dict_2, state_dict_base, low_precision, metric
+            )
+        except Exception as e:
+            logging.error(f"Error calculating {metric}: {str(e)}")
+            results[metric] = f"Error calculating {metric}: {str(e)}"
+
+    return results
+
+
+def calculate_metrics_by_split(
+    state_dict_1: dict,
+    state_dict_2: dict,
+    state_dict_base: dict,
+    low_precision: bool,
+    metric: str,
+) -> str:
+    """
+    Calculate metrics for each key and integrate results.
+
+    Args:
+        state_dict_1 (dict): State dictionary of first model
+        state_dict_2 (dict): State dictionary of second model
+        state_dict_base (dict): State dictionary of base model
+        low_precision (bool): Whether to use 8-bit quantization
+        metric (str): Metric to calculate ('pcc', 'ed', 'cs')
+
+    Returns:
+        str: Integrated metric result as formatted string
+    """
+    total_similarity = 0.0
+    total_weight = 0.0
+    split_results = {}
+
+    # Determine the number of layers
+    num_layers = state_dict_base["lm_head.weight"].shape[0]
+
+    # Check architectures
+    if (
+        state_dict_1["lm_head.weight"].shape[0]
+        != state_dict_2["lm_head.weight"].shape[0]
+    ):
+        shape_1 = state_dict_1["lm_head.weight"].shape
+        shape_2 = state_dict_2["lm_head.weight"].shape
+        logging.warning(
+            f"Warning: Model architectures do not match. "
+            f"Using sub weight space instead.\n"
+            f"Vocab sizes in model 1: {shape_1[0]}, "
+            f"Vocab sizes in model 2: {shape_2[0]}"
+        )
+
+    # Process each key
+    for key, base_params in tqdm(
+        state_dict_base.items(), desc=f"Processing {metric.upper()} by key"
+    ):
+        try:
+            if key not in state_dict_1 or key not in state_dict_2:
+                logging.warning(f"Key {key} not found in one of the models")
+                continue
+
+            # Get parameters and calculate deltas
+            params_1 = state_dict_1[key][:num_layers]
+            params_2 = state_dict_2[key][:num_layers]
+
+            delta_1 = (params_1 - base_params).view(-1)
+            delta_2 = (params_2 - base_params).view(-1)
+
+            if low_precision:
+                delta_1 = quantize_8bit(delta_1)
+                delta_2 = quantize_8bit(delta_2)
+
+            # Calculate weight based on parameter count
+            weight = delta_1.numel()
+
+            # Calculate metric for current key
+            if metric == "pcc":
+                stack = torch.stack((delta_1, delta_2), dim=0)
+                split_similarity = torch.corrcoef(stack)[0, 1].item()
+            elif metric == "ed":
+                split_similarity = torch.dist(delta_1, delta_2).item()
+            elif metric == "cs":
+                split_similarity = cosine_similarity(delta_1, delta_2)
+            else:
+                raise ValueError(f"Unsupported metric: {metric}")
+
+            # Skip NaN values
+            if torch.isnan(torch.tensor(split_similarity)):
+                logging.warning(f"Skipping key {key} due to NaN result")
+                continue
+
+            # Store valid result
+            split_results[key] = split_similarity
+
+            # Update weighted average only for valid results
+            weight = delta_1.numel()
+            total_similarity += split_similarity * weight
+            total_weight += weight
+
+            # Log progress for large layers
+            if weight > 1000000:
+                logging.info(
+                    f"Layer {key}: {metric.upper()} = {split_similarity:.4f}, parameters = {weight}"
+                )
+
+            # Free memory
+            del delta_1, delta_2
+
+        except Exception as e:
+            logging.error(f"Error processing key {key}: {str(e)}")
+            continue
+
+    # Calculate final weighted average
+    if total_weight > 0:
+        final_result = total_similarity / total_weight
+
+        # Log summary statistics
+        logging.info(f"\nSummary for {metric.upper()}:")
+        logging.info(f"Total parameters: {total_weight}")
+
+        # Log detailed results for valid splits
+        logging.info(f"\nDetailed {metric.upper()} results by key:")
+        for key, value in split_results.items():
+            logging.info(f"{key}: {value:.4f}")
+
+        metric_names = {
+            "pcc": "Pearson Correlation Coefficient",
+            "ed": "Euclidean Distance",
+            "cs": "Cosine Similarity",
+        }
+
+        return f"Model Kinship based on {metric_names[metric]} (weighted average): {final_result:.4f}"
+    else:
+        return f"Error: No valid parameters found for {metric.upper()} calculation"

fusion_bench-0.2.30/fusion_bench/metrics/model_kinship/utility.py

@@ -0,0 +1,184 @@
+import logging
+from enum import Enum
+from typing import List
+
+import click
+import torch
+from tqdm import tqdm
+from transformers import (
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    PretrainedConfig,
+)
+
+
+class Metric(str, Enum):
+    """Enumeration of supported metrics"""
+
+    PCC = "pcc"
+    ED = "ed"
+    CS = "cs"
+
+    @classmethod
+    def list(cls) -> List[str]:
+        """Return list of supported metric values"""
+        return [metric.value for metric in cls]
+
+
+def get_config(model: str, trust_remote_code: bool = False) -> PretrainedConfig:
+    """
+    Fetch the configuration of a pretrained model from HuggingFace.
+
+    Args:
+        model (str): The name or path of the model to load configuration for.
+        trust_remote_code (bool, optional): Whether to trust remote code during loading.
+                                            Defaults to False.
+
+    Returns:
+        PretrainedConfig: The configuration object of the specified model.
+    """
+    # Fetch the configuration from HuggingFace's model hub.
+    config = AutoConfig.from_pretrained(
+        model,
+        trust_remote_code=trust_remote_code,  # Whether to allow remote code execution.
+    )
+    return config
+
+
+def validate_models(model_1: str, model_2: str, base_model: str) -> None:
+    """
+    Validate model names to ensure they are different and exist.
+
+    Args:
+        model_1: Name of the first model
+        model_2: Name of the second model
+        base_model: Name of the base model
+
+    Raises:
+        click.BadParameter: If validation fails
+    """
+    if model_1 == model_2 or model_1 == base_model or model_2 == base_model:
+        raise click.BadParameter("All model names must be different")
+
+
+def quantize_8bit(x: torch.Tensor) -> torch.Tensor:
+    # Get absolute min and max values
+    abs_max = torch.max(torch.abs(x))
+
+    # Scale to [-127, 127] range for 8-bit signed integers
+    # Using 127 instead of 128 to keep zero exactly representable
+    scaled = 127 * (x / abs_max)
+
+    # Round to nearest integer
+    quantized = torch.round(scaled)
+
+    # Clamp values to ensure they stay in valid range
+    quantized = torch.clamp(quantized, -127, 127)
+
+    return quantized
+
+
+def load_model_state_dict(model_name: str, device: str) -> dict:
+    """
+    Load a model and return its state dictionary.
+
+    Args:
+        model_name (str): Name or path of the model to load
+        device (str): Device to load the model on ('cuda' or 'cpu')
+
+    Returns:
+        dict: State dictionary of the loaded model
+    """
+    logging.info(f"Loading model: {model_name}")
+    model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
+    state_dict = model.state_dict()
+    del model  # Free memory
+    return state_dict
+
+
+def extract_delta_parameters(
+    model_1_name: str,
+    model_2_name: str,
+    model_base_name: str,
+    low_precision: bool,
+    device: str = "cuda" if torch.cuda.is_available() else "cpu",
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """
+    Extract the delta parameters (weight differences) between two models
+    relative to a base model.
+
+    Args:
+        model_1_name (str): Name or path of the first model.
+        model_2_name (str): Name or path of the second model.
+        model_base_name (str): Name or path of the base model for comparison.
+        low_precision (bool): Whether to use low precision weights
+
+    Returns:
+        (torch.Tensor, torch.Tensor): Delta parameters of model_1 and model_2 relative to base model.
+    """
+
+    # Extract state dictionaries from models
+    state_dict_1 = load_model_state_dict(model_1_name, device)
+    state_dict_2 = load_model_state_dict(model_2_name, device)
+    state_dict_base = load_model_state_dict(model_base_name, device)
+
+    # Determine the number of layers
+    num_layers = state_dict_base["lm_head.weight"].shape[0]
+
+    # Check if model architectures match, log a warning if not
+    if (
+        state_dict_1["lm_head.weight"].shape[0]
+        != state_dict_2["lm_head.weight"].shape[0]
+    ):
+        shape_1 = state_dict_1["lm_head.weight"].shape
+        shape_2 = state_dict_2["lm_head.weight"].shape
+        logging.warning(
+            f"Warning: Model architectures do not match. "
+            f"Using sub weight space instead.\n"
+            f"Vocab sizes in model 1: {shape_1[0]}, "
+            f"Vocab sizes in model 2: {shape_2[0]}"
+        )
+
+    # Initialize lists to store delta parameters for both models
+    d_vector_1, d_vector_2 = [], []
+
+    # Iterate over keys in the base model's state dictionary with tqdm
+    for key, base_params in tqdm(
+        state_dict_base.items(), desc="Processing keys", unit="key"
+    ):
+        # Only proceed if key exists in both models
+        try:
+            if key not in state_dict_1 or key not in state_dict_2:
+                logging.warning(f"Key {key} not found in one of the models")
+                continue
+        except Exception as e:
+            logging.error(f"Error processing key {key}: {str(e)}")
+
+        # Get the parameters for each model (truncate to num_layers for consistency)
+        params_1 = state_dict_1[key][:num_layers]
+        params_2 = state_dict_2[key][:num_layers]
+
+        # Compute the deltas relative to the base model
+        delta_1 = (params_1 - base_params).view(-1)
+        delta_2 = (params_2 - base_params).view(-1)
+
+        # Accumulate deltas
+        d_vector_1.append(delta_1)
+        d_vector_2.append(delta_2)
+
+    # Clear memory
+    del state_dict_1, state_dict_2, state_dict_base
+
+    logging.info("Concatenating delta vectors...")
+
+    d_vector_1 = torch.cat(d_vector_1)
+    d_vector_2 = torch.cat(d_vector_2)
+
+    if low_precision:
+        logging.info("Quantizing delta vectors to 8-bit precision...")
+        d_vector_1 = quantize_8bit(d_vector_1)
+        d_vector_2 = quantize_8bit(d_vector_2)
+        logging.info("Quantization complete")
+
+    return d_vector_1, d_vector_2

{fusion_bench-0.2.29 → fusion_bench-0.2.30}/fusion_bench/models/masks/mask_model.py

@@ -113,21 +113,27 @@ class MaskModel(ParameterDictModel):
     def get_distribution(
         self,
         mask_type: Literal["discrete", "continuous"],
+        temperature: float = 0.5,
         **kwargs,
     ):
         return {
-            name: self._param_to_distribution(param, mask_type=mask_type, **kwargs)
+            name: self._param_to_distribution(
+                param, mask_type=mask_type, temperature=temperature, **kwargs
+            )
             for name, param in self.named_parameters()
         }
 
     def sample_mask(
         self,
         mask_type: Literal["discrete", "continuous"] = "discrete",
+        temperature: float = 0.5,
         **kwargs,
     ):
         mask = {}
         for name, param in self.named_parameters():
-            dist = self._param_to_distribution(param, mask_type, **kwargs)
+            dist = self._param_to_distribution(
+                param, mask_type, temperature=temperature, **kwargs
+            )
             if mask_type == "discrete":
                 mask[name] = dist.sample()
             elif mask_type == "continuous":

{fusion_bench-0.2.29 → fusion_bench-0.2.30}/fusion_bench/models/open_clip/modeling.py

@@ -1,3 +1,10 @@
+from fusion_bench.utils.packages import is_open_clip_available
+
+if not is_open_clip_available():
+    raise ImportError(
+        "open_clip is not installed. Please install it with `pip install open_clip_torch`."
+    )
+
 from typing import Callable, List
 
 import open_clip