fusion-bench 0.2.15__py3-none-any.whl → 0.2.16__py3-none-any.whl

fusion_bench_config/method/randes/superposed_task_arithmetic.yaml

@@ -0,0 +1,20 @@
+name: superposed_task_arithmetic
+#* === base randes options ===
+mode: random_binary_diagonal_matrix
+# weights for all mlp and attn layers
+target_layer: 
+  - mlp_w
+  - attn_w
+random_seed: 42 # for random_binary_diagonal_matrix
+different_across_layers: True
+joint_matrix_mode: flatten_hstack
+rank: 1 # for columnwise svd
+random_components: False
+shift_layers: 0
+debug: 0
+verbose: 0
+dropout_rate: 1
+#* === task arithmetic options ===
+scaling_factor: 0.5
+# path to save/load the model
+model_path: null

fusion_bench_config/method/randes/superposed_task_arithmetic_lora.yaml

@@ -0,0 +1,20 @@
+_target_: fusion_bench.method.randes.SuperposedTaskArithmeticLoRAAlgorithm
+#* === base randes options ===
+mode: random_binary_diagonal_matrix
+# weights for all mlp and attn layers
+target_layer:
+  - mlp_w
+  - attn_w
+random_seed: 42 # for random_binary_diagonal_matrix
+different_across_layers: True
+joint_matrix_mode: flatten_hstack
+rank: 1 # for columnwise svd
+random_components: False
+shift_layers: 0
+debug: 0
+verbose: 0
+dropout_rate: 1
+#* === task arithmetic options ===
+scaling_factor: 0.5
+# path to save/load the model
+model_path: null

fusion_bench_config/method/sparselo_pruning/llama_iterative_sparselo.yaml

@@ -1,10 +1,11 @@
-_target_: fusion_bench.method.losparse.sparselo.IterativeSparseLoForLlama
+_target_: fusion_bench.method.sparselo.sparselo.IterativeSparseLoForLlama
 _recursive_: false
 nsamples: 128
 seed: 0
 rank: 128
 num_iterations: 10
 variant: wanda
+use_reference_model: false
 # `prune_type` can be either `unstructured` or `semistructured`
 prune_type: unstructured
 # device and dtype to compute the pruning mask

fusion_bench_config/method/sparselo_pruning/llama_pcp_sparselo.yaml

@@ -1,4 +1,4 @@
-_target_: fusion_bench.method.losparse.sparselo.PCPSparseLoForLlama
+_target_: fusion_bench.method.sparselo.sparselo.PCPSparseLoForLlama
 _recursive_: false
 nsamples: 128
 seed: 0

fusion_bench_config/method/sparselo_pruning/llama_sparselo.yaml

@@ -1,4 +1,4 @@
-_target_: fusion_bench.method.losparse.sparselo.SparseLoForLlama
+_target_: fusion_bench.method.sparselo.sparselo.SparseLoForLlama
 _recursive_: false
 nsamples: 128
 seed: 0

fusion_bench_config/method/tall_mask/task_arithmetic.yaml

@@ -0,0 +1,4 @@
+_target_: fusion_bench.method.tall_mask.TallMaskTaskArithmeticAlgorithm
+tall_mask_lambda: 0.6
+debug: 0
+verbose: 0

fusion_bench_config/model/clip-vit/clip-vit-base-patch32_TALL10.yaml

@@ -0,0 +1,29 @@
+# The 20 tasks used in the paper: 
+# Wang et al. Localizing Task Information for Improved Model Merging and Compression
+# http://arxiv.org/abs/2405.07813
+defaults:
+  # pre-trained model
+  - clip-vit-base-patch32
+  # eight tasks in the task arithmetic paper
+  - clip-vit-base-patch32_sun397
+  - clip-vit-base-patch32_stanford-cars
+  - clip-vit-base-patch32_resisc45
+  - clip-vit-base-patch32_eurosat
+  - clip-vit-base-patch32_svhn
+  - clip-vit-base-patch32_gtsrb
+  - clip-vit-base-patch32_mnist
+  - clip-vit-base-patch32_dtd
+  # additional 6 tasks in the TALL mask paper (TALL 14)
+  - clip-vit-base-patch32_oxford_flowers102
+  - clip-vit-base-patch32_pcam
+  # - clip-vit-base-patch32_fer2013
+  # - clip-vit-base-patch32_oxford-iiit-pet
+  # - clip-vit-base-patch32_stl10
+  # - clip-vit-base-patch32_cifar100
+  # additional 6 tasks in the TALL mask paper (TALL 20)
+  # - clip-vit-base-patch32_cifar10
+  # - clip-vit-base-patch32_food101
+  # - clip-vit-base-patch32_fashion_mnist
+  # - clip-vit-base-patch32_emnist_letters
+  # - clip-vit-base-patch32_kmnist
+  # - clip-vit-base-patch32_rendered-sst2

fusion_bench_config/model/clip-vit/clip-vit-base-patch32_TALL12.yaml

@@ -0,0 +1,29 @@
+# The 20 task used in the paper: 
+# Wang et al. Localizing Task Information for Improved Model Merging and Compression
+# http://arxiv.org/abs/2405.07813
+defaults:
+  # pre-trained model
+  - clip-vit-base-patch32
+  # eight tasks in the task arithmetic paper
+  - clip-vit-base-patch32_sun397
+  - clip-vit-base-patch32_stanford-cars
+  - clip-vit-base-patch32_resisc45
+  - clip-vit-base-patch32_eurosat
+  - clip-vit-base-patch32_svhn
+  - clip-vit-base-patch32_gtsrb
+  - clip-vit-base-patch32_mnist
+  - clip-vit-base-patch32_dtd
+  # additional 6 tasks in the TALL mask paper (TALL 14)
+  - clip-vit-base-patch32_oxford_flowers102
+  - clip-vit-base-patch32_pcam
+  - clip-vit-base-patch32_fer2013
+  - clip-vit-base-patch32_oxford-iiit-pet
+  # - clip-vit-base-patch32_stl10
+  # - clip-vit-base-patch32_cifar100
+  # additional 6 tasks in the TALL mask paper (TALL 20)
+  # - clip-vit-base-patch32_cifar10
+  # - clip-vit-base-patch32_food101
+  # - clip-vit-base-patch32_fashion_mnist
+  # - clip-vit-base-patch32_emnist_letters
+  # - clip-vit-base-patch32_kmnist
+  # - clip-vit-base-patch32_rendered-sst2

fusion_bench_config/model/clip-vit/clip-vit-base-patch32_TALL16.yaml

@@ -0,0 +1,29 @@
+# The 20 task used in the paper: 
+# Wang et al. Localizing Task Information for Improved Model Merging and Compression
+# http://arxiv.org/abs/2405.07813
+defaults:
+  # pre-trained model
+  - clip-vit-base-patch32
+  # eight tasks in the task arithmetic paper
+  - clip-vit-base-patch32_sun397
+  - clip-vit-base-patch32_stanford-cars
+  - clip-vit-base-patch32_resisc45
+  - clip-vit-base-patch32_eurosat
+  - clip-vit-base-patch32_svhn
+  - clip-vit-base-patch32_gtsrb
+  - clip-vit-base-patch32_mnist
+  - clip-vit-base-patch32_dtd
+  # additional 6 tasks in the TALL mask paper (TALL 14)
+  - clip-vit-base-patch32_oxford_flowers102
+  - clip-vit-base-patch32_pcam
+  - clip-vit-base-patch32_fer2013
+  - clip-vit-base-patch32_oxford-iiit-pet
+  - clip-vit-base-patch32_stl10
+  - clip-vit-base-patch32_cifar100
+  # additional 6 tasks in the TALL mask paper (TALL 20)
+  - clip-vit-base-patch32_cifar10
+  - clip-vit-base-patch32_food101
+  # - clip-vit-base-patch32_fashion_mnist
+  # - clip-vit-base-patch32_emnist_letters
+  # - clip-vit-base-patch32_kmnist
+  # - clip-vit-base-patch32_rendered-sst2

fusion_bench_config/model/clip-vit/clip-vit-base-patch32_TALL18.yaml

@@ -0,0 +1,29 @@
+# The 20 task used in the paper: 
+# Wang et al. Localizing Task Information for Improved Model Merging and Compression
+# http://arxiv.org/abs/2405.07813
+defaults:
+  # pre-trained model
+  - clip-vit-base-patch32
+  # eight tasks in the task arithmetic paper
+  - clip-vit-base-patch32_sun397
+  - clip-vit-base-patch32_stanford-cars
+  - clip-vit-base-patch32_resisc45
+  - clip-vit-base-patch32_eurosat
+  - clip-vit-base-patch32_svhn
+  - clip-vit-base-patch32_gtsrb
+  - clip-vit-base-patch32_mnist
+  - clip-vit-base-patch32_dtd
+  # additional 6 tasks in the TALL mask paper (TALL 14)
+  - clip-vit-base-patch32_oxford_flowers102
+  - clip-vit-base-patch32_pcam
+  - clip-vit-base-patch32_fer2013
+  - clip-vit-base-patch32_oxford-iiit-pet
+  - clip-vit-base-patch32_stl10
+  - clip-vit-base-patch32_cifar100
+  # additional 6 tasks in the TALL mask paper (TALL 20)
+  - clip-vit-base-patch32_cifar10
+  - clip-vit-base-patch32_food101
+  - clip-vit-base-patch32_fashion_mnist
+  - clip-vit-base-patch32_emnist_letters
+  # - clip-vit-base-patch32_kmnist
+  # - clip-vit-base-patch32_rendered-sst2

fusion_bench_config/modelpool/CLIPVisionModelPool/clip-vit-base-patch32_TALL10.yaml

@@ -0,0 +1,8 @@
+# The 20 task used in the paper: 
+# Wang et al. Localizing Task Information for Improved Model Merging and Compression
+# http://arxiv.org/abs/2405.07813
+defaults:
+  - CLIPVisionModelPool@: _template
+  - /model/clip-vit@models: clip-vit-base-patch32_TALL10
+  - /dataset/image_classification/train@train_datasets: TALL10
+  - /dataset/image_classification/test@test_datasets: TALL10

fusion_bench_config/modelpool/CLIPVisionModelPool/clip-vit-base-patch32_TALL12.yaml

@@ -0,0 +1,8 @@
+# The 20 task used in the paper: 
+# Wang et al. Localizing Task Information for Improved Model Merging and Compression
+# http://arxiv.org/abs/2405.07813
+defaults:
+  - CLIPVisionModelPool@: _template
+  - /model/clip-vit@models: clip-vit-base-patch32_TALL12
+  - /dataset/image_classification/train@train_datasets: TALL12
+  - /dataset/image_classification/test@test_datasets: TALL12

fusion_bench_config/modelpool/CLIPVisionModelPool/clip-vit-base-patch32_TALL16.yaml

@@ -0,0 +1,8 @@
+# The 20 task used in the paper: 
+# Wang et al. Localizing Task Information for Improved Model Merging and Compression
+# http://arxiv.org/abs/2405.07813
+defaults:
+  - CLIPVisionModelPool@: _template
+  - /model/clip-vit@models: clip-vit-base-patch32_TALL16
+  - /dataset/image_classification/train@train_datasets: TALL16
+  - /dataset/image_classification/test@test_datasets: TALL16

fusion_bench_config/modelpool/CLIPVisionModelPool/clip-vit-base-patch32_TALL18.yaml

@@ -0,0 +1,8 @@
+# The 20 task used in the paper: 
+# Wang et al. Localizing Task Information for Improved Model Merging and Compression
+# http://arxiv.org/abs/2405.07813
+defaults:
+  - CLIPVisionModelPool@: _template
+  - /model/clip-vit@models: clip-vit-base-patch32_TALL18
+  - /dataset/image_classification/train@train_datasets: TALL18
+  - /dataset/image_classification/test@test_datasets: TALL18

fusion_bench_config/modelpool/CausalLMPool/deepseek-v2-lite.yaml

@@ -0,0 +1,15 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+
+pretrained_model_name_or_path: deepseek-ai/DeepSeek-V2-Lite
+
+models:
+  _pretrained_:
+    _target_: fusion_bench.models.modeling_deepseek_v2.DeepseekV2ForCausalLM.from_pretrained
+    pretrained_model_name_or_path: ${...pretrained_model_name_or_path}
+    torch_dtype: bfloat16
+    device_map: auto
+    trust_remote_code: true
+
+tokenizer:
+  _target_: transformers.AutoTokenizer.from_pretrained
+  pretrained_model_name_or_path: ${..pretrained_model_name_or_path}

fusion_bench_config/modelpool/CausalLMPool/mixtral-8x7b.yaml

@@ -0,0 +1,14 @@
+_target_: fusion_bench.modelpool.CausalLMPool
+
+pretrained_model_name_or_path: mistralai/Mixtral-8x7B-v0.1
+
+models:
+  _pretrained_:
+    _target_: transformers.AutoModelForCausalLM.from_pretrained
+    pretrained_model_name_or_path: ${...pretrained_model_name_or_path}
+    torch_dtype: bfloat16
+    device_map: auto
+
+tokenizer:
+  _target_: transformers.AutoTokenizer.from_pretrained
+  pretrained_model_name_or_path: ${..pretrained_model_name_or_path}

fusion_bench_config/modelpool/SeqenceClassificationModelPool/roberta-base_glue.yaml

@@ -0,0 +1,69 @@
+defaults:
+  - Seq2SeqLMPool@: _template
+  - /model/roberta@models:
+      - roberta_base
+      - roberta_glue-cola
+      - roberta_glue-mnli
+      - roberta_glue-mrpc
+      - roberta_glue-qnli
+      - roberta_glue-qqp
+      - roberta_glue-rte
+      - roberta_glue-sst2
+      - roberta_glue-stsb
+# _target_: fusion_bench.modelpool.SequenceClassificationModelPool
+# _recursive_: false
+
+_dataset_loader: fusion_bench.tasks.flan_t5_text_generation.glue_load_dataset.load_glue_dataset
+test_datasets:
+  glue-cola:
+    _target_: ${..._dataset_loader}
+    _recursive_: false
+    name: cola
+    tokenizer: ${...tokenizer}
+    split: validation
+  glue-mnli:
+    _target_: ${..._dataset_loader}
+    _recursive_: false
+    name: mnli
+    tokenizer: ${...tokenizer}
+    split: validation_matched
+  glue-mrpc:
+    _target_: ${..._dataset_loader}
+    _recursive_: false
+    name: mrpc
+    tokenizer: ${...tokenizer}
+    split: validation
+  glue-qnli:
+    _target_: ${..._dataset_loader}
+    _recursive_: false
+    name: qnli
+    tokenizer: ${...tokenizer}
+    split: validation
+  glue-qqp:
+    _target_: ${..._dataset_loader}
+    _recursive_: false
+    name: qqp
+    tokenizer: ${...tokenizer}
+    split: validation
+  glue-rte:
+    _target_: ${..._dataset_loader}
+    _recursive_: false
+    name: rte
+    tokenizer: ${...tokenizer}
+    split: validation
+  glue-sst2:
+    _target_: ${..._dataset_loader}
+    _recursive_: false
+    name: sst2
+    tokenizer: ${...tokenizer}
+    split: validation
+  glue-stsb:
+    _target_: ${..._dataset_loader}
+    _recursive_: false
+    name: stsb
+    tokenizer: ${...tokenizer}
+    split: validation
+
+tokenizer:
+  _target_: transformers.AutoTokenizer.from_pretrained
+  pretrained_model_name_or_path: roberta-base