fusion-bench 0.2.29__py3-none-any.whl → 0.2.30__py3-none-any.whl

fusion_bench/method/__init__.py

@@ -144,7 +144,15 @@ _extra_objects = {
 
 if TYPE_CHECKING:
     from .ada_svd import AdaSVDMergingForCLIPVisionModel
-    from .adamerging import *
+    from .adamerging import (
+        CLIPLayerWiseAdaMergingAlgorithm,
+        CLIPTaskWiseAdaMergingAlgorithm,
+        FlanT5LayerWiseAdaMergingAlgorithm,
+        GPT2LayerWiseAdaMergingAlgorithm,
+        LayerWiseAdaMergingForLlamaSFT,
+        ResNetLayerWiseAdamerging,
+        ResNetTaskWiseAdamerging,
+    )
     from .analysis import TaskVectorCosSimilarity, TaskVectorViolinPlot
     from .base_algorithm import BaseAlgorithm, BaseModelFusionAlgorithm
     from .bitdelta import BitDeltaAlgorithm

fusion_bench/method/base_algorithm.py

@@ -40,6 +40,7 @@ from typing import Optional  # noqa: F401
 
 from fusion_bench.mixins import BaseYAMLSerializable
 from fusion_bench.modelpool import BaseModelPool
+from fusion_bench.utils.misc import DeprecationWarningMeta
 
 __all__ = ["BaseAlgorithm", "BaseModelFusionAlgorithm"]
 
@@ -202,27 +203,36 @@ class BaseAlgorithm(BaseYAMLSerializable):
         pass
 
 
-BaseModelFusionAlgorithm = BaseAlgorithm
-"""
-Alias for BaseAlgorithm class.
+# Create a deprecated wrapper class that inherits from BaseAlgorithm
+class BaseModelFusionAlgorithm(BaseAlgorithm, metaclass=DeprecationWarningMeta):
+    """
+    Alias for BaseAlgorithm class.
 
-This alias is provided for backward compatibility and semantic clarity.
-Some users may prefer the more explicit name 'BaseModelFusionAlgorithm'
-to emphasize that this class is specifically designed for model fusion
-tasks, while others may prefer the shorter 'BaseAlgorithm' name.
+    .. deprecated::
+        BaseModelFusionAlgorithm is deprecated and will be removed in a future version.
+        Use :class:`BaseAlgorithm` instead.
 
-Both names refer to the exact same class and can be used interchangeably.
+    This alias was provided for backward compatibility and semantic clarity.
+    Both names refer to the same base class and can be used interchangeably,
+    but BaseAlgorithm is now the preferred name for all implementations.
 
-Examples:
-    Using the original name:
-    >>> class MyAlgorithm(BaseAlgorithm):
-    ...     def run(self, modelpool): pass
+    Examples:
+        Preferred (using BaseAlgorithm):
 
-    Using the alias:
-    >>> class MyAlgorithm(BaseModelFusionAlgorithm):
-    ...     def run(self, modelpool): pass
+        >>> class MyAlgorithm(BaseAlgorithm):
+        ...     def run(self, modelpool): pass
 
-Note:
-    The alias is maintained for compatibility but BaseAlgorithm is the
-    preferred name for new implementations.
-"""
+        Deprecated (using BaseModelFusionAlgorithm):
+
+        >>> class MyAlgorithm(BaseModelFusionAlgorithm):  # Will trigger deprecation warning
+        ...     def run(self, modelpool): pass
+
+    Note:
+        New implementations should use :class:`BaseAlgorithm` exclusively.
+        The BaseModelFusionAlgorithm alias will be removed in a future release.
+
+    Warning:
+        Using BaseModelFusionAlgorithm will trigger a DeprecationWarning.
+    """
+
+    pass

fusion_bench/metrics/model_kinship/__init__.py

@@ -0,0 +1,2 @@
+# Exploring Model Kinship for Merging LLMs
+# The implementation of this module is borrowed from: https://github.com/zjunlp/ModelKinship/

fusion_bench/metrics/model_kinship/calculate.py

@@ -0,0 +1,77 @@
+import logging
+from typing import List
+
+import numpy
+import torch
+
+from .utility import Metric
+
+
+def cosine_similarity(a, b):
+    similarity = numpy.sqrt(numpy.dot(a, b) ** 2 / (numpy.dot(a, a) * numpy.dot(b, b)))
+    return similarity
+
+
+def calculate_model_kinship(
+    delta1: numpy.ndarray, delta2: numpy.ndarray, metrics: List[str]
+) -> dict:
+    """
+    Calculate model kinship using specified metrics.
+
+    Args:
+        delta1: Delta parameters for first model
+        delta2: Delta parameters for second model
+        metrics: List of metrics to calculate
+
+    Returns:
+        dict: Dictionary of metric names and their calculated values
+    """
+    results = {}
+    for metric in metrics:
+        try:
+            if metric not in Metric.list():
+                raise ValueError(f"Unsupported metric: {metric}")
+            results[metric] = calculate_metric(delta1, delta2, metric)
+        except Exception as e:
+            results[metric] = f"Error calculating {metric}: {str(e)}"
+    return results
+
+
+def calculate_metric(
+    d_vector_1: torch.Tensor, d_vector_2: torch.Tensor, metric: str
+) -> str:
+    """
+    Calculate the specified metric between two delta vectors.
+
+    Args:
+        d_vector_1 (torch.Tensor): Delta parameters for model 1.
+        d_vector_2 (torch.Tensor): Delta parameters for model 2.
+        metric (str): The metric to calculate ('pcc', 'ed', 'cs').
+
+    Returns:
+        str: A formatted string with the result of the chosen metric.
+    """
+    logging.info(f"Starting calculation of {metric.upper()} metric...")
+
+    # Pearson Correlation Coefficient (PCC)
+    if metric == "pcc":
+        # Stack the two vectors and calculate the Pearson correlation coefficient
+        stack = torch.stack((d_vector_1, d_vector_2), dim=0)
+        pcc = torch.corrcoef(stack)[0, 1].item()
+        return f"Model Kinship based on Pearson Correlation Coefficient: {pcc}"
+
+    # Euclidean Distance (ED)
+    elif metric == "ed":
+        # Compute the Euclidean distance between the vectors
+        distance = torch.dist(d_vector_1, d_vector_2).item()
+        return f"Model Kinship based on Euclidean Distance: {distance}"
+
+    # Cosine Similarity (CS)
+    elif metric == "cs":
+        # Compute cosine similarity
+        cs = cosine_similarity(d_vector_1, d_vector_2)
+        return f"Model Kinship based on Cosine Similarity: {cs}"
+
+    # If metric is not recognized
+    else:
+        return "Invalid metric specified."

fusion_bench/metrics/model_kinship/calculate_split.py

@@ -0,0 +1,171 @@
+import logging
+from typing import Dict, List
+
+import numpy
+import torch
+from tqdm import tqdm
+
+from .utility import Metric, load_model_state_dict, quantize_8bit
+
+
+def cosine_similarity(a, b):
+    similarity = numpy.sqrt(numpy.dot(a, b) ** 2 / (numpy.dot(a, a) * numpy.dot(b, b)))
+    return similarity
+
+
+def calculate_model_kinship_split(
+    model_1_name: str,
+    model_2_name: str,
+    model_base_name: str,
+    low_precision: bool,
+    metrics: List[str],
+    device: str = "cuda" if torch.cuda.is_available() else "cpu",
+) -> dict:
+
+    # Extract state dictionaries from models
+    state_dict_1 = load_model_state_dict(model_1_name, device)
+    state_dict_2 = load_model_state_dict(model_2_name, device)
+    state_dict_base = load_model_state_dict(model_base_name, device)
+    results = {}
+
+    # Validate metrics before processing
+    valid_metrics = Metric.list()
+    for metric in metrics:
+        try:
+            if metric not in valid_metrics:
+                raise ValueError(
+                    f"Unsupported metric: {metric}. Valid metrics are: {', '.join(valid_metrics)}"
+                )
+            results[metric] = calculate_metrics_by_split(
+                state_dict_1, state_dict_2, state_dict_base, low_precision, metric
+            )
+        except Exception as e:
+            logging.error(f"Error calculating {metric}: {str(e)}")
+            results[metric] = f"Error calculating {metric}: {str(e)}"
+
+    return results
+
+
+def calculate_metrics_by_split(
+    state_dict_1: dict,
+    state_dict_2: dict,
+    state_dict_base: dict,
+    low_precision: bool,
+    metric: str,
+) -> str:
+    """
+    Calculate metrics for each key and integrate results.
+
+    Args:
+        state_dict_1 (dict): State dictionary of first model
+        state_dict_2 (dict): State dictionary of second model
+        state_dict_base (dict): State dictionary of base model
+        low_precision (bool): Whether to use 8-bit quantization
+        metric (str): Metric to calculate ('pcc', 'ed', 'cs')
+
+    Returns:
+        str: Integrated metric result as formatted string
+    """
+    total_similarity = 0.0
+    total_weight = 0.0
+    split_results = {}
+
+    # Determine the number of layers
+    num_layers = state_dict_base["lm_head.weight"].shape[0]
+
+    # Check architectures
+    if (
+        state_dict_1["lm_head.weight"].shape[0]
+        != state_dict_2["lm_head.weight"].shape[0]
+    ):
+        shape_1 = state_dict_1["lm_head.weight"].shape
+        shape_2 = state_dict_2["lm_head.weight"].shape
+        logging.warning(
+            f"Warning: Model architectures do not match. "
+            f"Using sub weight space instead.\n"
+            f"Vocab sizes in model 1: {shape_1[0]}, "
+            f"Vocab sizes in model 2: {shape_2[0]}"
+        )
+
+    # Process each key
+    for key, base_params in tqdm(
+        state_dict_base.items(), desc=f"Processing {metric.upper()} by key"
+    ):
+        try:
+            if key not in state_dict_1 or key not in state_dict_2:
+                logging.warning(f"Key {key} not found in one of the models")
+                continue
+
+            # Get parameters and calculate deltas
+            params_1 = state_dict_1[key][:num_layers]
+            params_2 = state_dict_2[key][:num_layers]
+
+            delta_1 = (params_1 - base_params).view(-1)
+            delta_2 = (params_2 - base_params).view(-1)
+
+            if low_precision:
+                delta_1 = quantize_8bit(delta_1)
+                delta_2 = quantize_8bit(delta_2)
+
+            # Calculate weight based on parameter count
+            weight = delta_1.numel()
+
+            # Calculate metric for current key
+            if metric == "pcc":
+                stack = torch.stack((delta_1, delta_2), dim=0)
+                split_similarity = torch.corrcoef(stack)[0, 1].item()
+            elif metric == "ed":
+                split_similarity = torch.dist(delta_1, delta_2).item()
+            elif metric == "cs":
+                split_similarity = cosine_similarity(delta_1, delta_2)
+            else:
+                raise ValueError(f"Unsupported metric: {metric}")
+
+            # Skip NaN values
+            if torch.isnan(torch.tensor(split_similarity)):
+                logging.warning(f"Skipping key {key} due to NaN result")
+                continue
+
+            # Store valid result
+            split_results[key] = split_similarity
+
+            # Update weighted average only for valid results
+            weight = delta_1.numel()
+            total_similarity += split_similarity * weight
+            total_weight += weight
+
+            # Log progress for large layers
+            if weight > 1000000:
+                logging.info(
+                    f"Layer {key}: {metric.upper()} = {split_similarity:.4f}, parameters = {weight}"
+                )
+
+            # Free memory
+            del delta_1, delta_2
+
+        except Exception as e:
+            logging.error(f"Error processing key {key}: {str(e)}")
+            continue
+
+    # Calculate final weighted average
+    if total_weight > 0:
+        final_result = total_similarity / total_weight
+
+        # Log summary statistics
+        logging.info(f"\nSummary for {metric.upper()}:")
+        logging.info(f"Total parameters: {total_weight}")
+
+        # Log detailed results for valid splits
+        logging.info(f"\nDetailed {metric.upper()} results by key:")
+        for key, value in split_results.items():
+            logging.info(f"{key}: {value:.4f}")
+
+        metric_names = {
+            "pcc": "Pearson Correlation Coefficient",
+            "ed": "Euclidean Distance",
+            "cs": "Cosine Similarity",
+        }
+
+        return f"Model Kinship based on {metric_names[metric]} (weighted average): {final_result:.4f}"
+    else:
+        return f"Error: No valid parameters found for {metric.upper()} calculation"

fusion_bench/metrics/model_kinship/utility.py

@@ -0,0 +1,184 @@
+import logging
+from enum import Enum
+from typing import List
+
+import click
+import torch
+from tqdm import tqdm
+from transformers import (
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    PretrainedConfig,
+)
+
+
+class Metric(str, Enum):
+    """Enumeration of supported metrics"""
+
+    PCC = "pcc"
+    ED = "ed"
+    CS = "cs"
+
+    @classmethod
+    def list(cls) -> List[str]:
+        """Return list of supported metric values"""
+        return [metric.value for metric in cls]
+
+
+def get_config(model: str, trust_remote_code: bool = False) -> PretrainedConfig:
+    """
+    Fetch the configuration of a pretrained model from HuggingFace.
+
+    Args:
+        model (str): The name or path of the model to load configuration for.
+        trust_remote_code (bool, optional): Whether to trust remote code during loading.
+                                            Defaults to False.
+
+    Returns:
+        PretrainedConfig: The configuration object of the specified model.
+    """
+    # Fetch the configuration from HuggingFace's model hub.
+    config = AutoConfig.from_pretrained(
+        model,
+        trust_remote_code=trust_remote_code,  # Whether to allow remote code execution.
+    )
+    return config
+
+
+def validate_models(model_1: str, model_2: str, base_model: str) -> None:
+    """
+    Validate model names to ensure they are different and exist.
+
+    Args:
+        model_1: Name of the first model
+        model_2: Name of the second model
+        base_model: Name of the base model
+
+    Raises:
+        click.BadParameter: If validation fails
+    """
+    if model_1 == model_2 or model_1 == base_model or model_2 == base_model:
+        raise click.BadParameter("All model names must be different")
+
+
+def quantize_8bit(x: torch.Tensor) -> torch.Tensor:
+    # Get absolute min and max values
+    abs_max = torch.max(torch.abs(x))
+
+    # Scale to [-127, 127] range for 8-bit signed integers
+    # Using 127 instead of 128 to keep zero exactly representable
+    scaled = 127 * (x / abs_max)
+
+    # Round to nearest integer
+    quantized = torch.round(scaled)
+
+    # Clamp values to ensure they stay in valid range
+    quantized = torch.clamp(quantized, -127, 127)
+
+    return quantized
+
+
+def load_model_state_dict(model_name: str, device: str) -> dict:
+    """
+    Load a model and return its state dictionary.
+
+    Args:
+        model_name (str): Name or path of the model to load
+        device (str): Device to load the model on ('cuda' or 'cpu')
+
+    Returns:
+        dict: State dictionary of the loaded model
+    """
+    logging.info(f"Loading model: {model_name}")
+    model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
+    state_dict = model.state_dict()
+    del model  # Free memory
+    return state_dict
+
+
+def extract_delta_parameters(
+    model_1_name: str,
+    model_2_name: str,
+    model_base_name: str,
+    low_precision: bool,
+    device: str = "cuda" if torch.cuda.is_available() else "cpu",
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """
+    Extract the delta parameters (weight differences) between two models
+    relative to a base model.
+
+    Args:
+        model_1_name (str): Name or path of the first model.
+        model_2_name (str): Name or path of the second model.
+        model_base_name (str): Name or path of the base model for comparison.
+        low_precision (bool): Whether to use low precision weights
+
+    Returns:
+        (torch.Tensor, torch.Tensor): Delta parameters of model_1 and model_2 relative to base model.
+    """
+
+    # Extract state dictionaries from models
+    state_dict_1 = load_model_state_dict(model_1_name, device)
+    state_dict_2 = load_model_state_dict(model_2_name, device)
+    state_dict_base = load_model_state_dict(model_base_name, device)
+
+    # Determine the number of layers
+    num_layers = state_dict_base["lm_head.weight"].shape[0]
+
+    # Check if model architectures match, log a warning if not
+    if (
+        state_dict_1["lm_head.weight"].shape[0]
+        != state_dict_2["lm_head.weight"].shape[0]
+    ):
+        shape_1 = state_dict_1["lm_head.weight"].shape
+        shape_2 = state_dict_2["lm_head.weight"].shape
+        logging.warning(
+            f"Warning: Model architectures do not match. "
+            f"Using sub weight space instead.\n"
+            f"Vocab sizes in model 1: {shape_1[0]}, "
+            f"Vocab sizes in model 2: {shape_2[0]}"
+        )
+
+    # Initialize lists to store delta parameters for both models
+    d_vector_1, d_vector_2 = [], []
+
+    # Iterate over keys in the base model's state dictionary with tqdm
+    for key, base_params in tqdm(
+        state_dict_base.items(), desc="Processing keys", unit="key"
+    ):
+        # Only proceed if key exists in both models
+        try:
+            if key not in state_dict_1 or key not in state_dict_2:
+                logging.warning(f"Key {key} not found in one of the models")
+                continue
+        except Exception as e:
+            logging.error(f"Error processing key {key}: {str(e)}")
+
+        # Get the parameters for each model (truncate to num_layers for consistency)
+        params_1 = state_dict_1[key][:num_layers]
+        params_2 = state_dict_2[key][:num_layers]
+
+        # Compute the deltas relative to the base model
+        delta_1 = (params_1 - base_params).view(-1)
+        delta_2 = (params_2 - base_params).view(-1)
+
+        # Accumulate deltas
+        d_vector_1.append(delta_1)
+        d_vector_2.append(delta_2)
+
+    # Clear memory
+    del state_dict_1, state_dict_2, state_dict_base
+
+    logging.info("Concatenating delta vectors...")
+
+    d_vector_1 = torch.cat(d_vector_1)
+    d_vector_2 = torch.cat(d_vector_2)
+
+    if low_precision:
+        logging.info("Quantizing delta vectors to 8-bit precision...")
+        d_vector_1 = quantize_8bit(d_vector_1)
+        d_vector_2 = quantize_8bit(d_vector_2)
+        logging.info("Quantization complete")
+
+    return d_vector_1, d_vector_2

fusion_bench/models/masks/mask_model.py

@@ -113,21 +113,27 @@ class MaskModel(ParameterDictModel):
     def get_distribution(
         self,
         mask_type: Literal["discrete", "continuous"],
+        temperature: float = 0.5,
         **kwargs,
     ):
         return {
-            name: self._param_to_distribution(param, mask_type=mask_type, **kwargs)
+            name: self._param_to_distribution(
+                param, mask_type=mask_type, temperature=temperature, **kwargs
+            )
             for name, param in self.named_parameters()
         }
 
     def sample_mask(
         self,
         mask_type: Literal["discrete", "continuous"] = "discrete",
+        temperature: float = 0.5,
         **kwargs,
     ):
         mask = {}
         for name, param in self.named_parameters():
-            dist = self._param_to_distribution(param, mask_type, **kwargs)
+            dist = self._param_to_distribution(
+                param, mask_type, temperature=temperature, **kwargs
+            )
             if mask_type == "discrete":
                 mask[name] = dist.sample()
             elif mask_type == "continuous":

fusion_bench/models/open_clip/modeling.py

@@ -1,3 +1,10 @@
+from fusion_bench.utils.packages import is_open_clip_available
+
+if not is_open_clip_available():
+    raise ImportError(
+        "open_clip is not installed. Please install it with `pip install open_clip_torch`."
+    )
+
 from typing import Callable, List
 
 import open_clip

fusion_bench/models/wrappers/layer_wise_fusion.py

@@ -173,6 +173,24 @@ class LayerWiseMergedModel(nn.Module, Generic[TorchModelType]):
 
     @property
     def forward_model(self):
+        """
+        Get a functional model with merged parameters.
+
+        Returns a partial function that applies the pretrained model with the current
+        merged state dictionary. This allows for efficient forward passes without
+        modifying the original model's parameters.
+
+        Returns:
+            Callable: A partial function that can be called with (args, kwargs) to
+                perform forward pass with merged parameters.
+
+        Example:
+            ```python
+            # Internal usage during forward pass
+            forward_fn = merged_model.forward_model
+            output = forward_fn(args=(x,), kwargs={})
+            ```
+        """
         return functools.partial(
             functional_call,
             self.pretrained_model,
@@ -181,10 +199,30 @@ class LayerWiseMergedModel(nn.Module, Generic[TorchModelType]):
             strict=self.strict,
         )
 
-    def merge_and_unload(self, task_vector_mask: Optional[Dict[str, Tensor]] = None):
+    def merge_and_unload(
+        self,
+        task_vector_mask: Optional[Dict[str, Tensor]] = None,
+        copy: bool = False,
+    ) -> TorchModelType:
+        """
+        Merge models and return the final merged model.
+
+        Args:
+            task_vector_mask (Optional[Dict[str, Tensor]], optional): Optional masks
+                for selective parameter merging. Defaults to None.
+            copy (bool, optional): Whether to return a deep copy of the pretrained model.
+                Defaults to False. If True, the original pretrained model remains unchanged.
+
+        Returns:
+            TorchModelType: The pretrained model with merged parameters loaded.
+        """
         self.merge_weights(task_vector_mask=task_vector_mask)
-        self.pretrained_model.load_state_dict(self._merged_state_dict)
-        return self.pretrained_model
+        if copy:
+            model = deepcopy(self.pretrained_model)
+        else:
+            model = self.pretrained_model
+        model.load_state_dict(self._merged_state_dict)
+        return model
 
     def merge_weights(self, task_vector_mask: Optional[Dict[str, Tensor]] = None):
         """

fusion_bench/models/wrappers/task_wise_fusion.py

@@ -16,6 +16,7 @@ outputs = merged_model(inputs)
 
 import functools
 import logging
+from copy import deepcopy
 from typing import Any, Callable, Dict, Generic, Iterator, List, Optional  # noqa: F401
 
 import torch
@@ -327,7 +328,11 @@ class TaskWiseMergedModel(nn.Module, Generic[TorchModelType]):
         self._merged_state_dict = state_dict
         return state_dict
 
-    def merge_and_unload(self, task_vector_mask: Optional[Dict[str, Tensor]] = None):
+    def merge_and_unload(
+        self,
+        task_vector_mask: Optional[Dict[str, Tensor]] = None,
+        copy: bool = False,
+    ) -> TorchModelType:
         """
         Merge models and return the final merged model.
 
@@ -338,6 +343,8 @@ class TaskWiseMergedModel(nn.Module, Generic[TorchModelType]):
         Args:
             task_vector_mask (Optional[Dict[str, Tensor]], optional): Optional masks
                 for selective parameter merging. Defaults to None.
+            copy (bool, optional): Whether to return a deep copy of the pretrained model.
+                Defaults to False. If True, the original pretrained model remains unchanged.
 
         Returns:
             TorchModelType: The pretrained model with merged parameters loaded.
@@ -363,8 +370,12 @@ class TaskWiseMergedModel(nn.Module, Generic[TorchModelType]):
             The original pretrained model parameters will be lost.
         """
         self.merge_weights(task_vector_mask=task_vector_mask)
-        self.pretrained_model.load_state_dict(self._merged_state_dict)
-        return self.pretrained_model
+        if copy:
+            model = deepcopy(self.pretrained_model)
+        else:
+            model = self.pretrained_model
+        model.load_state_dict(self._merged_state_dict)
+        return model
 
     def forward(self, *args, **kwargs):
         """

fusion_bench/scripts/cli.py

@@ -69,6 +69,20 @@ def main(cfg: DictConfig) -> None:
     """
     OmegaConf.resolve(cfg)
     program: BaseHydraProgram = instantiate(cfg)
+
+    # Validate that instantiation succeeded and returned an object with 'run' method
+    if not hasattr(program, "run") or not callable(getattr(program, "run")):
+        err_msg = (
+            f"Expected an object with a callable 'run' method, but got {type(program).__name__}. "
+            "Ensure that the configuration specifies a concrete program class with '_target_'."
+        )
+        if "_target_" not in cfg:
+            err_msg += "\nThe '_target_' field is missing from the root configuration."
+        else:
+            err_msg += f"\nFound '_target_': {cfg._target_}"
+        err_msg += f"\n\nConfiguration content:\n{cfg}"
+        raise TypeError(err_msg)
+
     program.run()
 
 

fusion_bench/utils/devices.py

@@ -32,11 +32,13 @@ def clear_cuda_cache():
     Clears the CUDA memory cache to free up GPU memory.
     Works only if CUDA is available.
     """
+
     gc.collect()
     if torch.cuda.is_available():
         torch.cuda.empty_cache()
+        gc.collect()
     else:
-        log.warning("CUDA is not available. No cache to clear.")
+        log.debug("CUDA is not available. No cache to clear.")
 
 
 def to_device(

fusion_bench/utils/instantiate_utils.py

@@ -14,8 +14,8 @@ from lightning_utilities.core.rank_zero import rank_zero_only
 from omegaconf import DictConfig, OmegaConf, SCMode
 from omegaconf._utils import is_structured_config
 from rich import print
-from rich.panel import Panel
-from rich.syntax import Syntax
+
+from fusion_bench.utils.rich_utils import print_bordered
 
 PRINT_FUNCTION_CALL = True
 """
@@ -67,12 +67,22 @@ def _resolve_callable_name(f: Callable[..., Any]) -> str:
     return full_name
 
 
-def _format_args_kwargs(args, kwargs):
+def _get_obj_str(obj: Any) -> str:
+    if isinstance(obj, (str, int, float, bool, type(None))):
+        return repr(obj)
+    else:
+        return f"'<{type(obj).__name__} object>'"
+
+
+def _format_args_kwargs(args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> str:
     result_strings = []
     if len(args) > 0:
-        result_strings.append(", ".join(repr(arg) for arg in args))
+        result_strings.append(", ".join(_get_obj_str(arg) for arg in args))
+
     if len(kwargs) > 0:
-        result_strings.append(", ".join(f"{k}={repr(v)}" for k, v in kwargs.items()))
+        result_strings.append(
+            ", ".join(f"{k}={_get_obj_str(v)}" for k, v in kwargs.items())
+        )
 
     if len(result_strings) == 0:
         return ""
@@ -145,14 +155,14 @@ def _call_target(
     if _partial_:
         if PRINT_FUNCTION_CALL and getattr(rank_zero_only, "rank", 0) == 0:
             call_str = f"functools.partial({_resolve_callable_name(_target_)}, {_format_args_kwargs(args, kwargs)})"
-            PRINT_FUNCTION_CALL_FUNC(
-                Panel(
-                    Syntax(call_str, "python", theme="monokai", word_wrap=True),
-                    title="Instantiate by calling partial",
-                    border_style="cyan",
-                )
+            print_bordered(
+                call_str,
+                code_style="python",
+                title=f"Instantiate by calling {'function' if not isinstance(_target_, type) else 'class'}",
+                style="cyan",
+                expand=False,
+                print_fn=PRINT_FUNCTION_CALL_FUNC,
             )
-
         if CATCH_EXCEPTION:
             try:
                 return functools.partial(_target_, *args, **kwargs)
@@ -169,12 +179,13 @@ def _call_target(
     else:
         if PRINT_FUNCTION_CALL and getattr(rank_zero_only, "rank", 0) == 0:
             call_str = f"{_resolve_callable_name(_target_)}({_format_args_kwargs(args, kwargs)})"
-            PRINT_FUNCTION_CALL_FUNC(
-                Panel(
-                    Syntax(call_str, "python", theme="monokai", word_wrap=True),
-                    title="Instantiate by calling function",
-                    border_style="green",
-                )
+            print_bordered(
+                call_str,
+                code_style="python",
+                title=f"Instantiate by calling {'function' if not isinstance(_target_, type) else 'class'}",
+                style="green",
+                expand=False,
+                print_fn=PRINT_FUNCTION_CALL_FUNC,
             )
         if CATCH_EXCEPTION:
             try:

fusion_bench/utils/misc.py

@@ -178,3 +178,19 @@ def validate_and_suggest_corrections(
     if matches:
         msg += f". Did you mean {', '.join(repr(m) for m in matches)}?"
     raise ValueError(msg)
+
+
+class DeprecationWarningMeta(type):
+    """
+    Metaclass that issues a deprecation warning whenever a class using it is instantiated.
+    """
+
+    def __call__(cls, *args, **kwargs):
+        import warnings
+
+        warnings.warn(
+            f"{cls.__name__} is deprecated and will be removed in a future version. ",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return super(DeprecationWarningMeta, cls).__call__(*args, **kwargs)

fusion_bench/utils/rich_utils.py

@@ -1,6 +1,6 @@
 import logging
 from pathlib import Path
-from typing import Sequence
+from typing import Optional, Sequence
 
 import rich
 import rich.syntax
@@ -19,6 +19,9 @@ from rich.text import Text
 from rich.traceback import install as install_rich_traceback
 
 from fusion_bench.utils import pylogger
+from fusion_bench.utils.packages import _is_package_available
+
+install_rich_traceback()
 
 log = pylogger.RankedLogger(__name__, rank_zero_only=True)
 
@@ -61,7 +64,31 @@ def display_available_styles():
     console.print(Columns(style_samples, equal=True, expand=False))
 
 
-def print_bordered(message, title=None, style="blue", code_style=None):
+def format_code_str(message: str, code_style="python"):
+    if code_style.lower() == "python" and _is_package_available("black"):
+        # Use black formatting for python code if black is available
+        import black
+
+        try:
+            message = black.format_str(message, mode=black.Mode())
+        except black.InvalidInput:
+            pass  # If black fails, use the original message
+
+    return message.strip()
+
+
+def print_bordered(
+    message,
+    title=None,
+    style="blue",
+    code_style=None,
+    *,
+    expand: bool = True,
+    theme: str = "monokai",
+    background_color: Optional[str] = "default",
+    print_fn=print,
+    format_code: bool = True,
+):
     """
     Print a message with a colored border.
 
@@ -73,12 +100,63 @@ def print_bordered(message, title=None, style="blue", code_style=None):
                                 Set to None for plain text. Defaults to "python".
     """
     if code_style:
-        content = Syntax(message, code_style, theme="monokai", word_wrap=True)
+        if format_code:
+            message = format_code_str(message, code_style)
+        content = Syntax(
+            message,
+            code_style,
+            word_wrap=True,
+            theme=theme,
+            background_color=background_color,
+        )
     else:
         content = Text(message)
 
-    panel = Panel(content, title=title, border_style=style)
-    print(panel)
+    panel = Panel(content, title=title, border_style=style, expand=expand)
+    print_fn(panel)
+
+
+def print_code(
+    message,
+    title=None,
+    code_style=None,
+    *,
+    expand: bool = True,
+    theme: str = "monokai",
+    background_color: Optional[str] = "default",
+    print_fn=print,
+):
+    """
+    Print code or plain text with optional syntax highlighting.
+
+    Args:
+        message (str): The message or code to print.
+        title (str, optional): Optional title associated with this output. Currently
+            not used by this function, but kept for API compatibility. Defaults to None.
+        code_style (str, optional): The language/lexer name for syntax highlighting
+            (for example, ``"python"``). If ``None``, the message is rendered as plain
+            text without syntax highlighting. Defaults to ``None``.
+        expand (bool, optional): Placeholder flag for API symmetry with other printing
+            helpers. It is not used in the current implementation. Defaults to True.
+        theme (str, optional): Name of the Rich syntax highlighting theme to use when
+            ``code_style`` is provided. Defaults to ``"monokai"``.
+        background_color (str, optional): Background color style to apply to the code
+            block when using syntax highlighting. Defaults to ``"default"``.
+        print_fn (Callable, optional): Function used to render the resulting Rich
+            object. Defaults to :func:`rich.print`.
+    """
+    if code_style:
+        content = Syntax(
+            message,
+            code_style,
+            word_wrap=True,
+            theme=theme,
+            background_color=background_color,
+        )
+    else:
+        content = Text(message)
+
+    print_fn(content)
 
 
 @rank_zero_only
@@ -95,6 +173,9 @@ def print_config_tree(
     ),
     resolve: bool = False,
     save_to_file: bool = False,
+    *,
+    theme: str = "monokai",
+    background_color: Optional[str] = "default",
 ) -> None:
     """Prints the contents of a DictConfig as a tree structure using the Rich library.
 
@@ -134,7 +215,14 @@ def print_config_tree(
         else:
             branch_content = str(config_group)
 
-        branch.add(rich.syntax.Syntax(branch_content, "yaml"))
+        branch.add(
+            rich.syntax.Syntax(
+                branch_content,
+                "yaml",
+                theme=theme,
+                background_color=background_color,
+            )
+        )
 
     # print config tree
     rich.print(tree)
@@ -145,6 +233,35 @@ def print_config_tree(
             rich.print(tree, file=file)
 
 
+@rank_zero_only
+def print_config_yaml(
+    cfg: DictConfig,
+    resolve: bool = False,
+    output_path: Optional[str] = False,
+    *,
+    theme: str = "monokai",
+    background_color: Optional[str] = "default",
+) -> None:
+    """
+    Prints the contents of a DictConfig as a YAML string using the Rich library.
+
+    Args:
+        cfg: A DictConfig composed by Hydra.
+        resolve: Whether to resolve reference fields of DictConfig. Default is ``False``.
+        output_path: Optional path to export the config YAML to. If provided, the file is written to this path.
+    """
+    config_yaml = OmegaConf.to_yaml(cfg, resolve=resolve)
+    syntax = rich.syntax.Syntax(
+        config_yaml, "yaml", theme=theme, background_color=background_color
+    )
+    rich.print(syntax)
+
+    if output_path:
+        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+        with open(Path(output_path), "w") as file:
+            rich.print(syntax, file=file)
+
+
 @rank_zero_only
 def enforce_tags(cfg: DictConfig, save_to_file: bool = False) -> None:
     """Prompts user to input tags from command line if no tags are provided in config.

{fusion_bench-0.2.29.dist-info → fusion_bench-0.2.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fusion-bench
-Version: 0.2.29
+Version: 0.2.30
 Summary: A Comprehensive Benchmark of Deep Model Fusion
 Author-email: Anke Tang <tang.anke@foxmail.com>
 Project-URL: Repository, https://github.com/tanganke/fusion_bench
@@ -255,9 +255,9 @@ First, create a new Python file for the algorithm in the `fusion_bench/method` d
 Following the naming convention, the file should be named `{method_name_or_class}/{variant}.py`.
 
 ```python
-from fusion_bench import BaseModelFusionAlgorithm, BaseModelPool
+from fusion_bench import BaseAlgorithm, BaseModelPool
 
-class DerivedModelFusionAlgorithm(BaseModelFusionAlgorithm):
+class DerivedModelFusionAlgorithm(BaseAlgorithm):
     """
     An example of a derived model fusion algorithm.
     """
@@ -265,7 +265,7 @@ class DerivedModelFusionAlgorithm(BaseModelFusionAlgorithm):
     # _config_mapping maps the attribution to the corresponding key in the configuration file.
     # this is optional and can be used to serialize the object to a configuration file.
     # `self.config.hyperparam_1` will be mapped to the attribute `hyperparam_attr_1`.
-    _config_mapping = BaseModelFusionAlgorithm._config_mapping | {
+    _config_mapping = BaseAlgorithm._config_mapping | {
         "hyperparam_attr_1": "hyperparam_1",
         "hyperparam_attr_2": "hyperparam_2",
     }
@@ -344,9 +344,9 @@ If you find this benchmark useful, please consider citing our work:
 ```bibtex
 @article{tang2024fusionbench,
   title={Fusionbench: A comprehensive benchmark of deep model fusion},
-  author={Tang, Anke and Shen, Li and Luo, Yong and Hu, Han and Du, Bo and Tao, Dacheng},
-  journal={arXiv preprint arXiv:2406.03280},
-  year={2024}
+  author={Tang, Anke and Shen, Li and Luo, Yong and Yang, Enneng and Hu, Han and Zhang, Lefei and Du, Bo and Tao, Dacheng},
+  journal={Journal of Machine Learning Research},
+  year={2025}
 }
 ```
 

{fusion_bench-0.2.29.dist-info → fusion_bench-0.2.30.dist-info}/RECORD

@@ -48,8 +48,8 @@ fusion_bench/dataset/llama/stanford_shp.py,sha256=6ueXKnFXIBBobacU1h5WxGLZrSOtBk
 fusion_bench/dataset/llama/ultrachat.py,sha256=Go7WvrDAYnm184fdazHGRYLbSY6Xd7jrESyQeUJtOww,1736
 fusion_bench/dataset/llama/wikitext.py,sha256=9ZHR-nMfXRumd3o-PIj3n7B83YlVeqpGkZ2zJs2B-9Y,2883
 fusion_bench/dataset/llama/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fusion_bench/method/__init__.py,sha256=sXjV8DDn3yXVjzsl6k-nMVx6EABQDjXjY3xK-I6nvr0,9527
-fusion_bench/method/base_algorithm.py,sha256=OnKSNPQ_nIdIWxryyblW_sko7uoEBN4lGh-eLkJ4kh4,9004
+fusion_bench/method/__init__.py,sha256=Set_2GWpmI3q_WvbV1hBUfa6GFiIuajyiZR2hRbfrN0,9811
+fusion_bench/method/base_algorithm.py,sha256=Pa3A7ON0YK3PJqFE77IY9dpQC-tQGJpX6kdf8IMnM_k,9453
 fusion_bench/method/dummy.py,sha256=hb1y6LR_geRZ5eRgGwt5zJUcHYorCeIbs5i76CvurUc,1031
 fusion_bench/method/ensemble.py,sha256=Bjzqxt-tUp5cawT1jIhqKswN5QH3bkYbmuI4LS4uTG0,3619
 fusion_bench/method/model_recombination.py,sha256=b2ku5wCrWd1QSZscIra4KlhLDxt04JjU30ItMNvpZ6g,5268
@@ -257,6 +257,10 @@ fusion_bench/method/wudi/wudi.py,sha256=HL3Y0MPjozp7NML_UNjIWWPbQDQxYH_WG_Buyrip
 fusion_bench/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fusion_bench/metrics/continual_learning/__init__.py,sha256=f-mkv4SpXTq5kiQVHbe2g0IPf4yLFgu1Dw7g2DOK6T4,57
 fusion_bench/metrics/continual_learning/backward_transfer.py,sha256=LCMWFFmBgWv7UIAJqiTaSvVvanx4qjnXIGuCMYvzmtc,559
+fusion_bench/metrics/model_kinship/__init__.py,sha256=-XWD0NR6Xz-p4oE8AKGoWrq-s1ayqWse7qLgNRENsaU,137
+fusion_bench/metrics/model_kinship/calculate.py,sha256=FoyBQuz3-q2NRfUW9w0dq9Tm51WG83iF_L_nHMOSI20,2447
+fusion_bench/metrics/model_kinship/calculate_split.py,sha256=_aTw7nfAZeEhiyqWlUkzwafQXLI3iDQMHdFy6ZMb88w,5797
+fusion_bench/metrics/model_kinship/utility.py,sha256=9iF9bWsJOFhhLqPMDyHyg-PAmat_zYUbud-umTfgBLs,5903
 fusion_bench/metrics/nyuv2/__init__.py,sha256=Ed1FQTJAxguJoorZLHIO-cSIgKYHHfqdf17J3o9_feI,1390
 fusion_bench/metrics/nyuv2/depth.py,sha256=xmUokztxyPrl90qtcoQaanti6DbFaIVqglAo3PDnEso,2851
 fusion_bench/metrics/nyuv2/loss.py,sha256=YKZSqycNyPWJV29Qa12--Wh87zZvtJcuUxUuiPbccpM,2529
@@ -330,7 +334,7 @@ fusion_bench/models/llama/model_utils/misc.py,sha256=3SJ7wk71zLMVF-AJEvQ_KCfFaMg
 fusion_bench/models/llama/model_utils/mod.py,sha256=xzNOgTRfOK9q8kml4Q2nmSOl23f33dE1tPi5zxgpWK0,1498
 fusion_bench/models/llama/model_utils/visual.py,sha256=wpqWqEASyA7WhJLCfC26h0Cdn5CXnwC1qPJUlSXggo4,8310
 fusion_bench/models/masks/__init__.py,sha256=vXG6jrBkDbPsnrX6nMEYAW1rQuGEWDgdjID7cKzXvrs,69
-fusion_bench/models/masks/mask_model.py,sha256=YXNZ_CGp6VPshZH__Znh6Z07BqOK53G-Ltc1LVy1E3I,5502
+fusion_bench/models/masks/mask_model.py,sha256=NDVhtuvZ10NUfTLEI_ONTKiceuSF-W7T9SEeUnyZFYQ,5680
 fusion_bench/models/model_card_templates/default.md,sha256=OoU83l1hip1gKsoA08hoKx-nCrOYbKaVTVCjK0pt9WY,1028
 fusion_bench/models/modeling_deepseek_v2/__init__.py,sha256=trXrhtKb_gIxXVo7wSZ-il5sLJtDTiNZezRrEt3M8zM,505
 fusion_bench/models/modeling_deepseek_v2/configuration_deepseek.py,sha256=TblFOCfNwaXUnXnD-sxFhSn5Df-_yy2LMcrth-sBPFI,10301
@@ -364,7 +368,7 @@ fusion_bench/models/nyuv2/lightning_module.py,sha256=SLtC0yL6455uKeb-o07MR6v-xE4
 fusion_bench/models/nyuv2/resnet.py,sha256=PcCfBhEsxm7W8cu3epBbIbCYFARPrPTamIa3TtUAVa0,14305
 fusion_bench/models/nyuv2/resnet_dilated.py,sha256=4EXB6vrBJS307YP6k-TRY1dFJ50LURcTuzqN4tZzYRk,3125
 fusion_bench/models/open_clip/__init__.py,sha256=zT2sGAT98Py5vXMckZF4aD8MYEICEWa2p7nRg4IrS0w,192
-fusion_bench/models/open_clip/modeling.py,sha256=34wKcbxe5xb6fzAVdIz0QcsSXs-8FQFUyqRNlIJso78,5556
+fusion_bench/models/open_clip/modeling.py,sha256=YOCsM1RfvhqJkUzwK9T4WqX1NW7LyAIi0UnN6ERQ-rk,5775
 fusion_bench/models/open_clip/utils.py,sha256=YM_vGQSxIDoB2euHG54hhRGIcINJfR0NxNT5U42KRCw,10394
 fusion_bench/models/open_clip/variables_and_paths.py,sha256=_OBcKvZwSGvYSmgKtXOuekEJI-btW94Ia-BQ9n4isfY,1231
 fusion_bench/models/smile_moe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -376,9 +380,9 @@ fusion_bench/models/surgery/__init__.py,sha256=tcUSi2m9GzGWfvRDQScIbdEbFBS_35gm9
 fusion_bench/models/surgery/surgerymodelwrapper.py,sha256=F8jX88K5zVWC6HsfN-nGNkEiPwNrN11ydyQQ1EZHehM,5133
 fusion_bench/models/wrappers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fusion_bench/models/wrappers/ensemble.py,sha256=T-DAKrAm-ciZwV6Hbt8uASbjtoQpHTlvVyan3rhk_8k,11632
-fusion_bench/models/wrappers/layer_wise_fusion.py,sha256=A7LjG0inL5oeEVOkJwEUDM15v4dpQnsCq2y9zA78R3k,11198
+fusion_bench/models/wrappers/layer_wise_fusion.py,sha256=T1sbujx_84Pj5yHFy5QqfipT6v3p96gUmnMgyy4lG0c,12560
 fusion_bench/models/wrappers/layer_wise_fusion_doge_ta.py,sha256=q5Hc4BtLpAawMbxsWJRL-8OR-x7994Jhr9IyN7vKZ9o,16930
-fusion_bench/models/wrappers/task_wise_fusion.py,sha256=ROLANdDq0bZ3sIROqIv3udPN8lzDdEwxD0Jonx-5ycw,17465
+fusion_bench/models/wrappers/task_wise_fusion.py,sha256=iCrevrkG4uTr3U8_hgT_xEY4epnEK0EJO8yg-uEMIUI,17836
 fusion_bench/optim/__init__.py,sha256=JS7J2VjrM2LdkiFCxuQnIuFwBsWiPyFb7QuEU6V2bPY,845
 fusion_bench/optim/exception.py,sha256=fMgo1heiqfGhuI5RIbf30BwWSShn5RQiyeb30QtfTI0,1607
 fusion_bench/optim/mezo.py,sha256=Vm4vMGh10Fhe28_9L1MK8r_U7DrurA8Liprh2_gn4_U,3646
@@ -392,7 +396,7 @@ fusion_bench/programs/base_program.py,sha256=Bl_bv8SawEUc-GBTtZFMoii0y-r-0hOXBAJ
 fusion_bench/programs/fabric_fusion_program.py,sha256=wIHNpLUw6uAXpAasJRAMWut55hF_EGFShxn70zRRvfk,12449
 fusion_bench/programs/fusion_program.py,sha256=qLyA3FHJUMM1L3mlYn4jlnZzv9OKguWM5aGGIoLts2I,11309
 fusion_bench/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fusion_bench/scripts/cli.py,sha256=kEWLEkZEBqUr1_-XTePzNC5NM8lwWvgUBf0Lcuk_FI8,2739
+fusion_bench/scripts/cli.py,sha256=t3YFuscJluxxNdXawW8FOaYH2fKn7m_6bXNlJ8KcZZg,3414
 fusion_bench/scripts/imgui.py,sha256=r9Glbfbwu3JCsX9TKQFwcHarvwA_G7ff0jWBUPW1S1U,7613
 fusion_bench/scripts/nyuv2_mtl_train.py,sha256=W1C45R9NdF4O-UjCx1bUxRTdFE0-FlRpwJHZ5gY18rI,3602
 fusion_bench/scripts/webui.py,sha256=ROvZUIj-hR4JLgCiWEKGc25LMtAjaMAZLJ5ckDYt-w4,21513
@@ -457,24 +461,24 @@ fusion_bench/tasks/flan_t5_text_generation/glue_prompt_templates.py,sha256=mKMTX
 fusion_bench/utils/__init__.py,sha256=EvrvupFGAzxll_jO0HYk1-I6jCHqDrIwZ5vswlR-9Pw,5149
 fusion_bench/utils/cache_utils.py,sha256=-bTZijQgl4BuAx0VSJFD-bSDOXuq3o0NkrOaiLiyofU,4795
 fusion_bench/utils/data.py,sha256=QAXpsvzHOgfAf6G_Pe2a5HOKUAP8Mxz77avujQI9Fd8,10027
-fusion_bench/utils/devices.py,sha256=6AkGcs3flt0FSo9yfEREuehoTrgcc65gkwpTWQy8XsI,9546
+fusion_bench/utils/devices.py,sha256=IyUBaWbnZGDsAxI97LEioUj-JIjYTzxQo_EhyKY3RZM,9566
 fusion_bench/utils/dict.py,sha256=ZCK0CRRT_B1Z18WY_GOYcmth7k5x9Jn1k7XhAVWRu98,1379
 fusion_bench/utils/dtype.py,sha256=z6UlPGF9dzG4Ik8rXGf59PJk_RKzG6Trp8O6wcBS9PU,4360
 fusion_bench/utils/expr.py,sha256=zwHNrtIbOMnIChU-0ZI5qLbDva8zvHbizL-4F2TwM14,2386
 fusion_bench/utils/fabric.py,sha256=qKcJ6Xj-6rEGy35dsUPHzxZT6az9RkSNcyBQl1uOv0M,6050
 fusion_bench/utils/functools.py,sha256=7_tYJ2WD88_2DDuOOj5aZz3cYuslYH5tsVyIgCeLtmk,1318
 fusion_bench/utils/hydra_utils.py,sha256=TklUDKDEZlg4keI-TEZiqh4gFjr9-61Rt1RMlqkoSGk,1174
-fusion_bench/utils/instantiate_utils.py,sha256=OXkfhq_o3Sgy5n3Psf-HI-dIfbK9oD2GBdfcx3gT63Q,17526
+fusion_bench/utils/instantiate_utils.py,sha256=UNfx188feTDrMSgp-ocLHetj6uD6axZcC46dRfBMtko,17884
 fusion_bench/utils/json.py,sha256=XZvEqBGpq-e0MaKkkX-1_PD8xMf6IDLAn4BrAF7IeiU,4552
 fusion_bench/utils/lazy_imports.py,sha256=s-1ABhPyyHs7gW4aodCzu3NySzILzTL7kVNZ0DZRXJA,6156
 fusion_bench/utils/lazy_state_dict.py,sha256=mJaiAtKB1vlNUAoQILnnCmU80FGJ8MSwmdPpmdhOyDE,22206
-fusion_bench/utils/misc.py,sha256=_7BaS9dNKyySGU0qmTmE0Tk8WK82TEm7IBJxVRkuEAw,5315
+fusion_bench/utils/misc.py,sha256=xntIUj4cwgx10y7Z1YqXT0zU4nDHfnKRK_M9biWgLH4,5780
 fusion_bench/utils/modelscope.py,sha256=P8fV6Eff8oP0LVGIFGbLvuk8MBteysN438djZ6ZEfE4,10699
 fusion_bench/utils/packages.py,sha256=m2E0ryIMI0NwWR9vUHkK9FtZEwA1G-A4dYOf87olli4,2217
 fusion_bench/utils/parameters.py,sha256=ufEDOYJwcQQxLfveK8hBAGwpu5J3LA_cTWiDgZ2zkJ0,11788
 fusion_bench/utils/path.py,sha256=piznok_znXkTY71VBwJrxBlXureYOdQnMfvqaZ26qvc,2643
 fusion_bench/utils/pylogger.py,sha256=1Uy_LkHkbrYdt1g5Ge_eAh2YoCJwn3U3Ndouz9sVA6g,3419
-fusion_bench/utils/rich_utils.py,sha256=3Z0di-1IOs3QoovF2frNA28ITVKWBLdm84zbXdTrM28,5924
+fusion_bench/utils/rich_utils.py,sha256=CJKL1vIHm2EznWa4e7ExmY5-lRtRRHLd7ZFPcn2acUs,9664
 fusion_bench/utils/set.py,sha256=_43ZvGKJ_BK9sUslsSNhi7xEfuAQuyj3vViImnGpnCY,134
 fusion_bench/utils/state_dict_arithmetic.py,sha256=bXO3zewO3KDzRmTaznlsnURIoSlcW5V5IhuXGtI_nxk,41234
 fusion_bench/utils/tensorboard.py,sha256=9fkgNYR9LM38nPNkudcxL9TjLUseW-280M0k2nLff7o,1669
@@ -488,7 +492,7 @@ fusion_bench/utils/plot/token_notebook.py,sha256=bsntXf46Zz_RavTxNiB9c3-KvHw7LFw
 fusion_bench/utils/strenum/__init__.py,sha256=id9ORi1uXrDxhbmVxitJ1KDwLS4H3AAwFpaK5h1cQzw,8531
 fusion_bench/utils/strenum/_name_mangler.py,sha256=o11M5-bURW2RBvRTYXFQIPNeqLzburdoWLIqk8X3ydw,3397
 fusion_bench/utils/strenum/_version.py,sha256=6JQRo9LcvODbCOeVFYQb9HNJ_J9XiG_Zbn8ws2A3BV8,18466
-fusion_bench-0.2.29.dist-info/licenses/LICENSE,sha256=nhnOJlw4CPuPVE0qvkGmxfFgHmKi-6nzXvTu8t0NUdg,1066
+fusion_bench-0.2.30.dist-info/licenses/LICENSE,sha256=nhnOJlw4CPuPVE0qvkGmxfFgHmKi-6nzXvTu8t0NUdg,1066
 fusion_bench_config/README.md,sha256=Lc8YSBJ5oxf9KV5kKDivJ9LRyGuraGQPmBbgbdVA-j4,703
 fusion_bench_config/clip-vit-base-patch32_robustness_corrupted.yaml,sha256=pZ5dFgg5n1W9cKdNyGNa7b4yPd4aQSu2iR2-yw9hhbY,442
 fusion_bench_config/fabric_model_fusion.yaml,sha256=kSQbhBsKypVFA3rmkdhY9BITnZWDXJof-I35t473_U0,2646
@@ -1015,8 +1019,8 @@ fusion_bench_config/taskpool/LMEvalHarnessTaskPool/lm_eval.yaml,sha256=3q-KMuFaM
 fusion_bench_config/taskpool/OpenCLIPVisionModelTaskPool/ViT-B-16_TA8.yaml,sha256=GjpiiRownrBCpl-TNwWRW2PYePbF-Cl99jlLNPrK5T4,1017
 fusion_bench_config/taskpool/OpenCLIPVisionModelTaskPool/ViT-B-32_TA8.yaml,sha256=WwiYMQKehtJixDPnu5o3vcWe4yJksXTWRqOzm3uVWXQ,1017
 fusion_bench_config/taskpool/OpenCLIPVisionModelTaskPool/ViT-L-14_TA8.yaml,sha256=xGRt0J9joXTzWUew6DvoYprAWlPXhaVFw5AX4im5VQw,1017
-fusion_bench-0.2.29.dist-info/METADATA,sha256=RivzHbrFvjc6WrrpTlsPwyCpUz8vw8Kc7GfxIwtIKxk,26292
-fusion_bench-0.2.29.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-fusion_bench-0.2.29.dist-info/entry_points.txt,sha256=iUQ8MCJvda7HP4vYh2n1Teoapb4G9PBVYZkAfcc5SHU,116
-fusion_bench-0.2.29.dist-info/top_level.txt,sha256=BuO4TL6iHL_2yPBUX9-LlIrHRczA_BNMIFwweK0PQEI,13
-fusion_bench-0.2.29.dist-info/RECORD,,
+fusion_bench-0.2.30.dist-info/METADATA,sha256=fcL0hcELjiXF7XmX4E2efcc_v1SrlSL9fsqQ7WCxyVM,26298
+fusion_bench-0.2.30.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fusion_bench-0.2.30.dist-info/entry_points.txt,sha256=iUQ8MCJvda7HP4vYh2n1Teoapb4G9PBVYZkAfcc5SHU,116
+fusion_bench-0.2.30.dist-info/top_level.txt,sha256=BuO4TL6iHL_2yPBUX9-LlIrHRczA_BNMIFwweK0PQEI,13
+fusion_bench-0.2.30.dist-info/RECORD,,