fusion-bench 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

fusion_bench_config/method/rankone_moe/rankone_moe.yaml

@@ -0,0 +1,26 @@
+name: ??? # this can be
+# the path for loading the model weights, if specified, skip the test-time adaptation training
+checkpoint: False
+# the path for saving the model weights.
+save_checkpoint: False
+router_hidden_layers: 1
+init_lambda: 0.3
+batch_reduce: true
+
+# device to compute svd
+svd_accelerator: cuda
+rank_k: 32 # How many experts are added to the pool per task?
+select_k: -1  # How many experts are selected from the pool to merge? Range is (1, rank_k*task_num). In particular -1: All the experts in the pool
+
+# learning rate
+lr: 1e-4
+optimizer: adam
+# this is overrided by `fabric.devices` if launched from the `fusion_bench` CLI.
+devices: 1
+batch_size: 16
+num_workers: 16
+max_steps: 1000 # default: 1000
+# if true, we will use the gradient accumulation across tasks to save memory
+use_grad_accumulate: true
+cache_dir: outputs
+fast_dev_run: ${fast_dev_run}

fusion_bench_config/method/regmean/clip_regmean.yaml

@@ -3,6 +3,7 @@ _target_: fusion_bench.method.RegMeanAlgorithmForCLIP
 exclude_param_names_regex: []
 # numbers of examples to compute regmean weights
 num_regmean_examples: 256
+weight_transpose: true
 # float, reduce non-diagonal elements in regmean weights by multiplying this scalar
 reduce_non_diagonal_ratio: 0.6
 dataloader_kwargs:

fusion_bench_config/method/surgery/adamerging_surgery.yaml

@@ -0,0 +1,27 @@
+# this option can be "clip_task_wise_adamerging"
+name: clip_layer_wise_adamerging_surgery
+# this weights can be a list of float, or a string that points to a *.np, *.pt file containing the weights
+# if weights is specified, skip the test-time adaptation training
+weights: null
+# learning rate
+optimizer: adam
+lr: 1e-3
+init_values: 0.3
+# if `clamp_weights` is true, the weights will be clamped to [0, 1]
+clamp_weights: false
+# arguments of `functional_call`
+tie_weights: true
+strict: false
+# this is overrided by `fabric.devices` if launched from the `fusion_bench` CLI.
+devices: 1
+batch_size: 16
+num_workers: 8
+max_steps: 1000
+fast_dev_run: ${fast_dev_run}
+# the path for saving the merging weights
+save_merging_weights: 'merging_weights.pt'
+cache_dir: outputs
+
+# parameters of Surgery
+eval_iterations: 200
+surgery_steps: 1000

fusion_bench_config/modelpool/CausalLMPool/llama_alpaca_cleaned.yaml

@@ -0,0 +1,21 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+
+pretrained_model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
+
+models:
+  _pretrained_:
+    _target_: transformers.AutoModelForCausalLM.from_pretrained
+    pretrained_model_name_or_path: ${...pretrained_model_name_or_path}
+    torch_dtype: bfloat16
+
+tokenizer:
+  _target_: transformers.AutoTokenizer.from_pretrained
+  pretrained_model_name_or_path: ${..pretrained_model_name_or_path}
+
+train_datasets:
+  alpaca-cleaned:
+    _target_: fusion_bench.dataset.llama.alpaca.load_tokenized_alpaca_dataset
+    tokenizer: ${...tokenizer}
+    path: "yahma/alpaca-cleaned"
+    split: train
+    cache_path: null

fusion_bench_config/modelpool/CausalLMPool/llama_codealpaca.yaml

@@ -0,0 +1,21 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+
+pretrained_model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
+
+models:
+  _pretrained_:
+    _target_: transformers.AutoModelForCausalLM.from_pretrained
+    pretrained_model_name_or_path: ${...pretrained_model_name_or_path}
+    torch_dtype: bfloat16
+
+tokenizer:
+  _target_: transformers.AutoTokenizer.from_pretrained
+  pretrained_model_name_or_path: ${..pretrained_model_name_or_path}
+
+train_datasets:
+  codealpaca:
+    _target_: fusion_bench.dataset.llama.alpaca.load_tokenized_alpaca_dataset
+    tokenizer: ${...tokenizer}
+    path: sahil2801/CodeAlpaca-20k
+    split: train
+    cache_path: null

fusion_bench_config/modelpool/CausalLMPool/llama_metamathqa.yaml

@@ -0,0 +1,19 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+
+pretrained_model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
+
+models:
+  _pretrained_:
+    _target_: transformers.AutoModelForCausalLM.from_pretrained
+    pretrained_model_name_or_path: ${...pretrained_model_name_or_path}
+    torch_dtype: bfloat16
+
+tokenizer:
+  _target_: transformers.AutoTokenizer.from_pretrained
+  pretrained_model_name_or_path: ${..pretrained_model_name_or_path}
+
+train_datasets:
+  metamathqa:
+    _target_: fusion_bench.dataset.llama.metamathqa.load_tokenized_metamathqa
+    tokenizer: ${...tokenizer}
+    cache_path: null

fusion_bench_config/modelpool/CausalLMPool/llama_ultrachat.yaml

@@ -0,0 +1,18 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+
+pretrained_model_name_or_path: meta-llama/Llama-3-1B-Instruct
+
+models:
+  _pretrained_:
+    _target_: transformers.AutoModelForCausalLM.from_pretrained
+    pretrained_model_name_or_path: ${...pretrained_model_name_or_path}
+    torch_dtype: bfloat16
+
+tokenizer:
+  _target_: transformers.AutoTokenizer.from_pretrained
+  pretrained_model_name_or_path: ${..pretrained_model_name_or_path}
+
+train_datasets:
+  ultrachat-200k:
+    _target_: fusion_bench.dataset.llama.ultrachat.load_tokenized_ultrachat_200k
+    tokenizer: ${...tokenizer}

fusion_bench_config/modelpool/SeqenceClassificationModelPool/llama_preference700k.yaml

@@ -0,0 +1,23 @@
+_target_: fusion_bench.modelpool.SeqenceClassificationModelPool
+
+pretrained_model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
+
+models:
+  _pretrained_:
+    _target_: fusion_bench.modelpool.seq_classification_lm.create_reward_model_from_pretrained
+    pretrained_model_name_or_path: ${...pretrained_model_name_or_path}
+    torch_dtype: bfloat16
+    use_flash_attention_2: true
+
+tokenizer:
+  _target_: transformers.AutoTokenizer.from_pretrained
+  pretrained_model_name_or_path: ${..pretrained_model_name_or_path}
+  pad_token: <|end_of_text|> # do not use eos token (<|eos_id|>) as padding token because it is used as the end of each content
+
+train_datasets:
+  preference_700k:
+    _target_: fusion_bench.dataset.llama.preference_700k.load_tokenized_preference_700k_for_rlhf
+    tokenizer: ${...tokenizer}
+    path: hendrydong/preference_700K
+    split: train
+    cache_path: null

fusion_bench_config/modelpool/SeqenceClassificationModelPool/single_reward_model.yaml

@@ -0,0 +1,14 @@
+_target_: fusion_bench.modelpool.SeqenceClassificationModelPool
+
+pretrained_model_name_or_path: fusion-bench/Llama-3.2-1B-Instruct_Bradly-Terry-RM_Preference-700k
+
+models:
+  _pretrained_:
+    _target_: transformers.AutoModelForSequenceClassification.from_pretrained
+    pretrained_model_name_or_path: ${...pretrained_model_name_or_path}
+    torch_dtype: bfloat16
+
+tokenizer:
+  _target_: transformers.AutoTokenizer.from_pretrained
+  pretrained_model_name_or_path: ${..pretrained_model_name_or_path}
+  pad_token: <|end_of_text|> # do not use eos token (<|eos_id|>) as padding token because it is used as the end of each content

fusion_bench_config/nyuv2_config.yaml

@@ -1,13 +1,17 @@
 defaults:
   - hydra: default
+  - fabric: auto
   - modelpool: nyuv2_modelpool
   - method: simple_average
   - taskpool: nyuv2_taskpool
   - _self_
+
+_target_: fusion_bench.programs.FabricModelFusionProgram
+_recursive_: false
+
 fast_dev_run: false # Run a single batch of data to test the model or method
 use_lightning: true # Use the fabric to run the experiment
 print_config: true # Print the configuration to the console
 save_report: false # path to save the result report
-fabric: null
 trainer:
   devices: 1

fusion_bench_config/taskpool/CLIPVisionModelTaskPool/clip_rankone_wemoe_clip-vit-classification_TA8.yaml

@@ -0,0 +1,18 @@
+defaults:
+  - CLIPVisionModelTaskPool@: _template
+  - /dataset/image_classification/test@test_datasets:
+      - sun397
+      - stanford-cars
+      - resisc45
+      - eurosat
+      - svhn
+      - gtsrb
+      - mnist
+      - dtd
+  - _self_
+
+_target_: fusion_bench.taskpool.RankoneWEMoECLIPVisionModelTaskPool
+
+# === layer-wise routing weights saving ===
+layer_wise_routing_weights_save_path: null
+layer_wise_routing_weights_max_num: 1000

fusion_bench_config/taskpool/reward_model_evaluation.yaml

@@ -0,0 +1,18 @@
+_target_: fusion_bench.taskpool.llama.reward_model.RewardModelEvaluationTaskPool
+
+test_datasets:
+  preference_700k:
+    _target_: fusion_bench.dataset.llama.preference_700k.load_tokenized_preference_700k_for_rlhf
+    tokenizer: ${...tokenizer}
+    path: hendrydong/preference_700K
+    split: train
+    cache_path: null
+
+dataloader_kwargs:
+  shuffle: False
+  batch_size: 16
+
+tokenizer: ${..modelpool.tokenizer}
+
+max_num_samples: 1000
+seed: 42

fusion_bench_config/llama_weighted_average.yaml

@@ -1,26 +0,0 @@
-defaults:
-  - example_config
-  - override method: weighted_average_for_llama
-  - override modelpool: llama_for_causallm
-  - _self_
-modelpool:
-  models:
-    # the pre-trained model (base model) is optional
-    # if not provided, the first model will be used as the base model
-    - name: _pretrained_
-      path: meta-llama/Meta-Llama-3-8B
-    - name: expert_1
-      path: meta-llama/Meta-Llama-3-8B
-    - name: expert_2
-      path: meta-llama/Meta-Llama-3-8B-Instruct
-method:
-  normalize: true # if true, the weights will be normalized before merging
-  weights: # List of weights for each model
-    - 0.5
-    - 0.5
-  # if true, only the backbone of the model will be merged and the head will be keeped as the pre-trained model (if the pre-trained model is provided, otherwise the head of the first model will be used)
-  # if false, the whole model will be merged
-  backbone_only: true
-  merged_model_save_path: null
-  save_tokenizer: true
-  push_to_hub: false