fusion-bench 0.2.20__py3-none-any.whl → 0.2.22__py3-none-any.whl

fusion_bench_config/_get_started/clip_evaluate_single_model.yaml

@@ -0,0 +1,21 @@
+_target_: fusion_bench.programs.FabricModelFusionProgram
+_recursive_: false
+method:
+  _target_: fusion_bench.method.DummyAlgorithm
+modelpool:
+  _target_: fusion_bench.modelpool.CLIPVisionModelPool
+  models:
+    _pretrained_: openai/clip-vit-base-patch32
+taskpool:
+  _target_: fusion_bench.taskpool.CLIPVisionModelTaskPool
+  test_datasets:
+    sun397:
+      _target_: datasets.load_dataset
+      path: tanganke/sun397
+      split: test
+    stanford-cars:
+      _target_: datasets.load_dataset
+      path: tanganke/stanford_cars
+      split: test
+  clip_model: openai/clip-vit-base-patch32
+  processor: openai/clip-vit-base-patch32

fusion_bench_config/_get_started/clip_simple_average.yaml

@@ -0,0 +1,23 @@
+_target_: fusion_bench.programs.FabricModelFusionProgram # (1)!
+_recursive_: false
+method: # (2)!
+  _target_: fusion_bench.method.SimpleAverageAlgorithm
+modelpool: # (3)!
+  _target_: fusion_bench.modelpool.CLIPVisionModelPool
+  models:
+    _pretrained_: openai/clip-vit-base-patch32
+    sun397: tanganke/clip-vit-base-patch32_sun397
+    stanford-cars: tanganke/clip-vit-base-patch32_stanford-cars
+taskpool: # (4)!
+  _target_: fusion_bench.taskpool.CLIPVisionModelTaskPool
+  test_datasets:
+    sun397:
+      _target_: datasets.load_dataset
+      path: tanganke/sun397
+      split: test
+    stanford-cars:
+      _target_: datasets.load_dataset
+      path: tanganke/stanford_cars
+      split: test
+  clip_model: openai/clip-vit-base-patch32
+  processor: openai/clip-vit-base-patch32

fusion_bench_config/_get_started/clip_task_arithmetic.yaml

@@ -0,0 +1,24 @@
+_target_: fusion_bench.programs.FabricModelFusionProgram
+_recursive_: false
+method:
+  _target_: fusion_bench.method.TaskArithmeticAlgorithm
+  scaling_factor: 0.7
+modelpool:
+  _target_: fusion_bench.modelpool.CLIPVisionModelPool
+  models:
+    _pretrained_: openai/clip-vit-base-patch32
+    sun397: tanganke/clip-vit-base-patch32_sun397
+    stanford-cars: tanganke/clip-vit-base-patch32_stanford-cars
+taskpool:
+  _target_: fusion_bench.taskpool.CLIPVisionModelTaskPool
+  test_datasets:
+    sun397:
+      _target_: datasets.load_dataset
+      path: tanganke/sun397
+      split: test
+    stanford-cars:
+      _target_: datasets.load_dataset
+      path: tanganke/stanford_cars
+      split: test
+  clip_model: openai/clip-vit-base-patch32
+  processor: openai/clip-vit-base-patch32

fusion_bench_config/_get_started/greeting_program.yaml

@@ -0,0 +1,4 @@
+_target_: fusion_bench._get_started.greeting_program.GreetingProgram
+message: "Welcome to FusionBench"
+name: "Developer"
+repeat_count: 3

fusion_bench_config/fabric/loggers/csv_logger.yaml

@@ -3,9 +3,9 @@ _target_: lightning.fabric.loggers.CSVLogger
 # for example, `outputs/logs/lightning_logs/version_0` and `outputs/logs/lightning_logs/version_1` by default
 
 # root directory for all logging
-root_dir: outputs/logs
+root_dir: ${path.log_dir}
 # the name of the experiment
-name: lightning_logs
-version: null
+name: ""
+version: ""
 prefix: ""
 flush_logs_every_n_steps: 100

fusion_bench_config/fabric/loggers/tensorboard_logger.yaml

@@ -3,9 +3,9 @@ _target_: lightning.fabric.loggers.TensorBoardLogger
 # for example, `outputs/logs/lightning_logs/version_0` and `outputs/logs/lightning_logs/version_1` by default
 
 # root directory for all logging
-root_dir: outputs/logs
+root_dir: ${path.log_dir}
 # the name of the experiment
-name: "lightning_logs"
-version: null
+name: ""
+version: ""
 sub_dir: null
 default_hp_metric: false

fusion_bench_config/fabric_model_fusion.yaml

@@ -1,19 +1,47 @@
+# =============================================================================
+# FusionBench Fabric Model Fusion Configuration
+# =============================================================================
+# This configuration file defines the settings for running model fusion experiments
+# using PyTorch Lightning Fabric framework within FusionBench.
+#
+# The configuration includes:
+#
+# - Hydra framework settings and overrides
+# - PyTorch Lightning Fabric configuration for distributed training
+# - Path management for data, outputs, and logs
+# - (core components) Model pool, fusion method, and task pool specifications
+# - Experiment execution parameters and debugging options
+#
+# =============================================================================
+# Hydra Configuration Defaults
+# =============================================================================
 defaults:
-  - hydra: default
-  - fabric: auto
-  # --- Model, Method, Task ---
-  - modelpool: CLIPVisionModelPool/clip-vit-base-patch32_TA8
-  - method: dummy
-  - taskpool: dummy
-  - _self_
+  - hydra: default # Hydra framework configuration
+  - fabric: auto # PyTorch Lightning Fabric auto-configuration
+  - path: default # Path management configuration
+  # --- Core Components ---
+  - modelpool: CLIPVisionModelPool/clip-vit-base-patch32_TA8 # Model pool specification
+  - method: dummy # Fusion method (placeholder)
+  - taskpool: dummy # Task pool specification (placeholder)
+  - _self_ # Self-reference for override priority
+# =============================================================================
+# Program Configuration
+# =============================================================================
 _target_: fusion_bench.programs.FabricModelFusionProgram
-_recursive_: false
-fast_dev_run: false # Run a single batch of data to test the model or method
-# Run the script without actually running the experiment, use with `print_config=true`.
-# You can also use `--cfg` or `-c` to show the configuration instead of running.
-dry_run: false
-print_config: true # Print the configuration to the console
-merged_model_save_path: null # path to save the merged model, use "{log_dir}" to refer to the logger directory, for example `merged_model_save_path=\{log_dir\}/merged_model`
-merged_model_save_kwargs: null
-report_save_path: "{log_dir}/program_report.json" # path to save the result report
-print_function_call: true # set to false if you don't want to print the details of instantiate calls
+_recursive_: false # Disable recursive instantiation
+# =============================================================================
+# Experiment Execution Settings
+# =============================================================================
+# Development and debugging options
+fast_dev_run: false # This option is for quick testing. For example, run single batch instead of full dataset
+dry_run: false # Show configuration without running experiment
+print_config: true # Display full configuration before execution
+print_function_call: true # Show detailed instantiation calls
+# =============================================================================
+# Output and Logging Configuration
+# =============================================================================
+# Model saving configuration
+merged_model_save_path: null # Path to save merged model.
+merged_model_save_kwargs: null # Additional kwargs for model saving.
+# Report generation
+report_save_path: "{log_dir}/program_report.json" # Experiment results report path

fusion_bench_config/hydra/default.yaml

@@ -2,7 +2,11 @@ defaults:
   - override help: fusion_bench_help
   - override job_logging: rich_logging
 run:
-  dir: outputs/${hydra.job.name}/${now:%Y-%m-%d_%H-%M-%S}
+  dir: ${path.log_dir}
 sweep:
-  dir: outputs/${hydra.job.name}/${now:%Y-%m-%d_%H-%M-%S}
+  dir: ${path.log_dir}
   subdir: ${hydra.job.num}
+job:
+  env_set:
+    HYDRA_FULL_ERROR: ${oc.env:HYDRA_FULL_ERROR,1}
+output_subdir: ""

fusion_bench_config/llama_full_finetune.yaml

@@ -1,6 +1,7 @@
 defaults:
   - hydra: default
   - fabric: llama_fsdp
+  - path: default
   # --- Model, Method, Task ---
   - method: lm_finetune/fullfinetune_sft.yaml
   - modelpool: CausalLMPool/llama_alpaca_cleaned.yaml

fusion_bench_config/method/adamerging/clip.yaml

@@ -1,5 +1,5 @@
 # this option can be "clip_task_wise_adamerging"
-name: ???
+name: clip_layer_wise_adamerging
 # this weights can be a list of float, or a string that points to a *.np, *.pt file containing the weights
 # if weights is specified, skip the test-time adaptation training
 weights: null

fusion_bench_config/method/bitdelta/bitdelta.yaml

@@ -0,0 +1,12 @@
+_target_: fusion_bench.method.bitdelta.BitDeltaAlgorithm
+save_dir: null
+save_full_model: false
+# training arguments
+lr: 1e-4
+batch_size: 4
+num_steps: 100
+# dataset arguments
+dataset_name: c4
+subset: en
+split: train
+max_length: 128

fusion_bench_config/method/depth_upscaling.yaml

@@ -1,5 +1,8 @@
 _target_: DepthUpscalingAlgorithm
-# this should be a list of integers or string, indicating the sequence of layers. If the entry is an integer, it will use the n-th layer of the model. If the entry is a string, it will use the layers specified by the string. The string should be a valid python expression that evaluates to a list of integers.
+# this should be a list of integers or string, indicating the sequence of layers. 
+# If the entry is an integer, it will use the n-th layer of the model. 
+# If the entry is a string, it will use the layers specified by the string. 
+# The string should be a valid python expression that evaluates to a list of integers.
 # for example, ["range(0,12)", "range(6,12)"] will use the first 12 layers and the last 6 layers of the model to construct the new model
 # [0, 2, 4, "range(6,12)"] will use the 1st, 3rd, 5th, and the 7th to 12th layers of the model to construct the new model
 layer_indices: null

fusion_bench_config/method/fisher_merging/clip_fisher_merging.yaml

@@ -7,7 +7,6 @@ normalize_fisher_weight: true
 minimal_fisher_weight: 1e-6
 # common choices: 256, 512, 1024, 2048
 num_fisher_examples: 256
-zeroshot_weights_cache_dir: outputs/cache/clip_zeroshot_weights
 dataloader_kwargs:
   batch_size: 32
   num_workers: 0

fusion_bench_config/method/linear/simple_average_for_llama.yaml

@@ -1,5 +1,6 @@
 _target_: fusion_bench.method.SimpleAverageForLlama
 # set `merge_backbone` to true if you has a base model and only want to merge the backbone of the experts
 # if `merge_backbone` is False, this is equivalent to `SimpleAverageAlgorithm`
-merge_backbone: true
-model_save_path: null
+merge_backbone: false
+model_save_path: ${path.log_dir}/checkpoint
+show_pbar: true

fusion_bench_config/method/smile_upscaling/causal_lm_upscaling.yaml

@@ -0,0 +1,21 @@
+# Generic SMILE Upscaling Configuration for CausalLM models
+# Supports: Qwen2, Llama, Mistral models
+# The model type will be auto-detected from the base model
+_target_: fusion_bench.method.smile_upscaling.causal_lm_upscaling.SmileCausalLMUpscalingAlgorithm
+
+# Device and computation settings
+device: cuda # device to put the models on
+accelerator: cuda # device to perform SVD on
+
+# Model upscaling parameters
+num_experts_per_tok: 1 # Number of experts to activate per token
+rank_of_router: 8 # Rank for router weights
+rank_of_expert: 64 # Rank for expert weights
+
+# Model saving settings
+model_save_path: ${path.log_dir}/checkpoint # Set to save the merged model
+model_dtype: null # Optional: convert to specific dtype after merging
+save_with_remote_code: true
+
+# Optional: Explicitly specify model type instead of auto-detection
+model_type: null # Options: "qwen2", "llama", "mistral", or null for auto-detection

fusion_bench_config/method/smile_upscaling/error_accumulation.yaml

@@ -0,0 +1,5 @@
+# Measure error accumulation
+_target_: fusion_bench.method.smile_upscaling.error_accumulation.ErrorAccumulationAnalysisForCLIP
+gate_k: 16
+k: 128
+top_k: 1

fusion_bench_config/method/smile_upscaling/projected_energy.yaml

@@ -0,0 +1,2 @@
+# Measure projected energy
+_target_: fusion_bench.method.smile_upscaling.projected_energy.ProjectedEnergyAnalysis

fusion_bench_config/method/smile_upscaling/smile_qwen2_upscaling.yaml

@@ -4,10 +4,11 @@ device: cpu
 # device to perform SVD on
 accelerator: cuda
 # path to save/load the model
-model_path: null
+model_save_path: ${path.log_dir}/checkpoint
 model_dtype: null
 # SmileMoE parameters
 num_experts_per_tok: 1
 rank_of_router: 8
 # if rank_of_expert < 0, dense expert is used.
 rank_of_expert: 64
+save_with_remote_code: true

fusion_bench_config/method/wemoe/flan_t5_weight_ensembling_moe.yaml

@@ -0,0 +1,20 @@
+_target_: fusion_bench.method.we_moe.flan_t5_we_moe.FlanT5WeightEnsemblingMoEAlgorithm
+# the path for loading the model weights, if specified, skip the test-time adaptation training
+checkpoint: False
+# the path for saving the model weights.
+save_checkpoint: False
+router_hidden_layers: 2
+init_lambda: 0.3
+batch_reduce: true
+# learning rate
+lr: 1e-4
+optimizer: adam
+# this is overrided by `fabric.devices` if launched from the `fusion_bench` CLI.
+devices: 1
+batch_size: 4
+num_workers: 0
+max_steps: 200
+# if true, we will use the gradient accumulation across tasks to save memory
+use_grad_accumulate: true
+cache_dir: outputs
+fast_dev_run: ${fast_dev_run}

fusion_bench_config/modelpool/CLIPVisionModelPool/_template.yaml

@@ -2,11 +2,8 @@ _usage_: |
   defaults:
     - CLIPVisionModelPool@: _template
 _target_: fusion_bench.modelpool.CLIPVisionModelPool
-_version_: "0.2"
 _recursive_: False
 models: ???
 train_datasets: null
 test_datasets: null
-processor:
-  _target_: transformers.CLIPProcessor.from_pretrained
-  pretrained_model_name_or_path: openai/clip-vit-base-patch32
+processor: openai/clip-vit-base-patch32

fusion_bench_config/modelpool/CLIPVisionModelPool/clip-vit-base-patch32_individual.yaml

@@ -1,10 +1,5 @@
-defaults:
-  - CLIPVisionModelPool@: _template
+_target_: fusion_bench.modelpool.CLIPVisionModelPool
+_recursive_: False
 models:
-  _pretrained_:
-    _target_: transformers.CLIPVisionModel.from_pretrained
-    pretrained_model_name_or_path: ${...base_model}
-processor:
-  _target_: transformers.CLIPProcessor.from_pretrained
-  pretrained_model_name_or_path: ${..base_model}
-base_model: openai/clip-vit-base-patch32
+  _pretrained_: openai/clip-vit-base-patch32
+processor: ${.models._pretrained_}

fusion_bench_config/modelpool/CLIPVisionModelPool/clip-vit-base-patch32_single_finetuned.yaml

@@ -3,5 +3,5 @@ _recursive_: False
 processor: openai/clip-vit-base-patch32
 models:
   _pretrained_: openai/clip-vit-base-patch32
-  stanford-cars: tanganke/clip-vit-base-patch32_stanford-cars
+  finetuned: tanganke/clip-vit-base-patch32_stanford-cars
 platform: hf

fusion_bench_config/modelpool/CLIPVisionModelPool/clip-vit-base-patch32_svhn_and_mnist.yaml

@@ -1,9 +1,3 @@
-defaults:
-  - CLIPVisionModelPool@: _template
-  - /model/clip-vit@models:
-      - clip-vit-base-patch32
-      - clip-vit-base-patch32_svhn
-      - clip-vit-base-patch32_mnist
 _target_: fusion_bench.modelpool.CLIPVisionModelPool
 _recursive_: False
 processor: openai/clip-vit-base-patch32

fusion_bench_config/modelpool/CLIPVisionModelPool/clip-vit-large-patch14_TA8.yaml

@@ -24,7 +24,7 @@ models:
   _pretrained_: openai/clip-vit-large-patch14
   sun397: tanganke/clip-vit-large-patch14_sun397
   stanford-cars: tanganke/clip-vit-large-patch14_stanford-cars
-  resisc45: tanganke/clip-vit-large-patch14_dtd
+  resisc45: tanganke/clip-vit-large-patch14_resisc45
   eurosat: tanganke/clip-vit-large-patch14_eurosat
   svhn: tanganke/clip-vit-large-patch14_svhn
   gtsrb: tanganke/clip-vit-large-patch14_gtsrb

fusion_bench_config/modelpool/CLIPVisionModelPool/clip-vit-large-patch14_TA8_model_only.yaml

@@ -5,7 +5,7 @@ models:
   _pretrained_: openai/clip-vit-large-patch14
   sun397: tanganke/clip-vit-large-patch14_sun397
   stanford-cars: tanganke/clip-vit-large-patch14_stanford-cars
-  resisc45: tanganke/clip-vit-large-patch14_dtd
+  resisc45: tanganke/clip-vit-large-patch14_resisc45
   eurosat: tanganke/clip-vit-large-patch14_eurosat
   svhn: tanganke/clip-vit-large-patch14_svhn
   gtsrb: tanganke/clip-vit-large-patch14_gtsrb

fusion_bench_config/modelpool/CausalLMPool/Qwen2.5-1.5B_math_and_coder.yaml

@@ -1,11 +1,11 @@
 _target_: fusion_bench.modelpool.CausalLMPool
 _recursive_: false
 
-load_lazy: false
+enable_lazy_loading: false
 models:
   _pretrained_: Qwen/Qwen2.5-1.5B
-  expert_1: Qwen/Qwen2.5-Math-1.5B
-  expert_2: Qwen/Qwen2.5-Coder-1.5B
+  math: Qwen/Qwen2.5-Math-1.5B
+  code: Qwen/Qwen2.5-Coder-1.5B
 model_kwargs:
   torch_dtype: bfloat16
 tokenizer: Qwen/Qwen2.5-1.5B

fusion_bench_config/modelpool/CausalLMPool/Qwen2.5-7B-math_and_coder.yaml

@@ -0,0 +1,9 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+_recursive_: false
+models:
+  _pretrained_: Qwen/Qwen2.5-7B
+  math: Qwen/Qwen2.5-Math-7B
+  code: Qwen/Qwen2.5-Coder-7B
+model_kwargs:
+  torch_dtype: bfloat16
+tokenizer: Qwen/Qwen2.5-7B

fusion_bench_config/modelpool/CausalLMPool/mistral-7b.yaml

@@ -0,0 +1,6 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+models:
+  _pretrained_: mistralai/Mistral-7B-v0.1
+tokenizer: ${.models._pretrained_}
+model_kwargs:
+  torch_dtype: bfloat16

fusion_bench_config/modelpool/CausalLMPool/mixtral_moe_merging.yaml

@@ -0,0 +1,10 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+models:
+  _pretrained_: path_to_your_pretrained_model
+  expert_1: path_to_your_expert_model_1
+  expert_2: path_to_your_expert_model_2
+  expert_3: path_to_your_expert_model_3
+  expert_4: path_to_your_expert_model_4
+tokenizer: ${.models._pretrained_}
+model_kwargs:
+  torch_dtype: bfloat16

fusion_bench_config/modelpool/CausalLMPool/qwen2_math_1.5B_and_R1.yaml

@@ -1,17 +1,9 @@
 _target_: fusion_bench.modelpool.CausalLMPool
 _recursive_: false
 models:
-  _pretrained_:
-    _target_: transformers.AutoModelForCausalLM.from_pretrained
-    pretrained_model_name_or_path: Qwen/Qwen2.5-1.5B
-  expert_1:
-    _target_: transformers.AutoModelForCausalLM.from_pretrained
-    pretrained_model_name_or_path: Qwen/Qwen2.5-Math-1.5B
-  expert_2:
-    _target_: transformers.AutoModelForCausalLM.from_pretrained
-    pretrained_model_name_or_path: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+  _pretrained_: Qwen/Qwen2.5-1.5B
+  expert_1: Qwen/Qwen2.5-Math-1.5B
+  expert_2: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
 model_kwargs:
   torch_dtype: bfloat16
-tokenizer:
-  _target_: transformers.AutoTokenizer.from_pretrained
-  pretrained_model_name_or_path: Qwen/Qwen2.5-1.5B
+tokenizer: Qwen/Qwen2.5-1.5B

fusion_bench_config/modelpool/CausalLMPool/simle_mixtral_exp_v4.yaml

@@ -1,20 +1,10 @@
 _target_: fusion_bench.modelpool.CausalLMPool
 _recursive_: false
 models:
-  _pretrained_:
-    _target_: transformers.AutoModelForCausalLM.from_pretrained
-    pretrained_model_name_or_path: mistralai/Mistral-7B-v0.1
-  expert_1:
-    _target_: transformers.AutoModelForCausalLM.from_pretrained
-    pretrained_model_name_or_path: meta-math/MetaMath-Mistral-7B
-  expert_2:
-    _target_: transformers.AutoModelForCausalLM.from_pretrained
-    pretrained_model_name_or_path: cognitivecomputations/dolphin-2.1-mistral-7b
-  expert_3:
-    _target_: transformers.AutoModelForCausalLM.from_pretrained
-    pretrained_model_name_or_path: uukuguy/speechless-code-mistral-7b-v1.0
+  _pretrained_: mistralai/Mistral-7B-v0.1
+  expert_1: meta-math/MetaMath-Mistral-7B
+  expert_2: cognitivecomputations/dolphin-2.1-mistral-7b
+  expert_3: uukuguy/speechless-code-mistral-7b-v1.0
 model_kwargs:
-  torch_dtype: float16
-tokenizer:
-  _target_: transformers.AutoTokenizer.from_pretrained
-  pretrained_model_name_or_path: mistralai/Mistral-7B-v0.1
+  torch_dtype: bfloat16
+tokenizer: mistralai/Mistral-7B-v0.1

fusion_bench_config/modelpool/CausalLMPool/vicuna-7b-v1.5.yaml

@@ -0,0 +1,8 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+_recursive_: false
+models:
+  _pretrained_: meta-llama/Llama-2-7b-hf
+  finetuned_model: lmsys/vicuna-7b-v1.5
+model_kwargs:
+  torch_dtype: bfloat16
+tokenizer: ${.models.finetuned_model}

fusion_bench_config/modelpool/{SeqenceClassificationModelPool → SequenceClassificationModelPool}/llama_preference700k.yaml

@@ -1,4 +1,4 @@
-_target_: fusion_bench.modelpool.SeqenceClassificationModelPool
+_target_: fusion_bench.modelpool.SequenceClassificationModelPool
 pretrained_model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
 models:
   _pretrained_:

fusion_bench_config/modelpool/{SeqenceClassificationModelPool → SequenceClassificationModelPool}/single_reward_model.yaml

@@ -1,4 +1,4 @@
-_target_: fusion_bench.modelpool.SeqenceClassificationModelPool
+_target_: fusion_bench.modelpool.SequenceClassificationModelPool
 pretrained_model_name_or_path: fusion-bench/Llama-3.2-1B-Instruct_Bradly-Terry-RM_Preference-700k
 models:
   _pretrained_:

fusion_bench_config/nyuv2_config.yaml

@@ -1,8 +1,10 @@
 defaults:
   - hydra: default
   - fabric: auto
-  - modelpool: nyuv2_modelpool
+  - path: default
+  # --- Model, Method, Task ---
   - method: simple_average
+  - modelpool: nyuv2_modelpool
   - taskpool: nyuv2_taskpool
   - _self_
 _target_: fusion_bench.programs.FabricModelFusionProgram

fusion_bench_config/nyuv2_mtl_train.yaml

@@ -1,5 +1,6 @@
 defaults:
   - hydra: default
+  - path: default
   - _self_
 fast_dev_run: false
 exp_name: null

fusion_bench_config/path/default.yaml

@@ -0,0 +1,28 @@
+# =============================================================================
+# FusionBench Path Configuration
+# =============================================================================
+# This configuration file defines the directory structure and path settings
+# used throughout the FusionBench framework for model fusion experiments.
+# All paths are configured using Hydra's variable interpolation syntax.
+# Root directory - uses FUSION_BENCH_PROJECT_ROOT env var or current directory
+#
+# By default:
+#
+# root_dir (defaults to current directory)
+# ├── outputs (output_dir)
+# │   ├── cache (cache_dir)
+# │   └── <config_name>
+# │       └── <timestamp> (log_dir)
+# └── data (data_dir)
+#
+root_dir: ${oc.env:FUSION_BENCH_PROJECT_ROOT,"."}
+# Output directory for experiment results and artifacts
+output_dir: ${.root_dir}/outputs
+# Data directory - uses FUSION_BENCH_DATA_DIR env var or root_dir/data
+data_dir: ${oc.env:FUSION_BENCH_DATA_DIR,${.root_dir}/data}
+# Cache directory - uses FUSION_BENCH_CACHE_DIR env var or output_dir/cache
+cache_dir: ${oc.env:FUSION_BENCH_CACHE_DIR,${.output_dir}/cache}
+# Log directory with timestamped subdirectories for each run
+log_dir: ${.output_dir}/${hydra:job.config_name}/${now:%Y-%m-%d_%H-%M-%S}
+# Current working directory at runtime
+work_dir: ${hydra:runtime.cwd}

fusion_bench_config/taskpool/CLIPVisionModelTaskPool/clip-vit-base-patch32_svhn_and_mnist.yaml

@@ -0,0 +1,24 @@
+defaults:
+  - /dataset/image_classification/test@test_datasets:
+      - svhn
+      - mnist
+_target_: fusion_bench.taskpool.CLIPVisionModelTaskPool
+_recursive_: false
+test_datasets: ??? # The datasets to evaluate the model on
+base_model: openai/clip-vit-base-patch32
+clip_model: ${.base_model} # The base model to use
+processor: ${.base_model} # The base model to use
+data_processor: ${.processor}
+dataloader_kwargs:
+  batch_size: 128 # The batch size for the data loader
+  num_workers: 8 # The number of worker processes for data loading
+  pin_memory: True # Whether to pin memory in data loader
+  drop_last: False # Whether to drop the last incomplete batch
+  shuffle: False # Whether to shuffle the data
+# === layer-wise feature saving ===
+# The path to save the features to, if none then the features are not saved
+# This is the path to a directory, the features of task `task_name` will be saved in `feature_save_path/task_name.csv`
+layer_wise_feature_save_path: null
+layer_wise_feature_first_token_only: true # Whether to save only the first token of the features
+# The maximum number of samples to save the features for
+layer_wise_feature_max_num: 1000

fusion_bench_config/method/adamerging.yaml

@@ -1,23 +0,0 @@
-# this option can be one of "clip_task_wise_adamerging" or "clip_layer_wise_adamerging"
-name: clip_layer_wise_adamerging
-# this weights can be a list of float, or a string that points to a *.np, *.pt file containing the weights
-# if weights is specified, skip the test-time adaptation training
-weights: null
-# learning rate
-optimizer: adam
-lr: 1e-3
-init_values: 0.3
-# if `clamp_weights` is true, the weights will be clamped to [0, 1]
-clamp_weights: false
-# arguments of `functional_call`
-tie_weights: true
-strict: false
-# this is overrided by `fabric.devices` if launched from the `fusion_bench` CLI.
-devices: 1
-batch_size: 16
-num_workers: 8
-max_steps: 1000
-fast_dev_run: ${fast_dev_run}
-# the path for saving the merging weights
-save_merging_weights: 'merging_weights.pt'
-cache_dir: outputs

fusion_bench_config/modelpool/mixtral_moe_merging.yaml

@@ -1,14 +0,0 @@
-type: AutoModelForCausalLMPool
-# each model should have a name and a path, and the model is loaded from the path
-# this is equivalent to `AutoModelForCausalLM.from_pretrained(path)`
-models:
-  - name: _pretrained_
-    path: path_to_your_pretrained_model
-  - name: expert_1
-    path: path_to_your_expert_model_1
-  - name: expert_2
-    path: path_to_your_expert_model_2
-  - name: expert_3
-    path: path_to_your_expert_model_3
-  - name: expert_4
-    path: path_to_your_expert_model_4

fusion_bench_config/modelpool/mixtral_moe_upscaling.yaml

@@ -1,6 +0,0 @@
-type: AutoModelForCausalLMPool
-# each model should have a name and a path, and the model is loaded from the path
-# this is equivalent to `AutoModelForCausalLM.from_pretrained(path)`
-models:
-  - name: _pretrained_
-    path: path_to_your_pretrained_model