fusion-bench 0.2.6__py3-none-any.whl → 0.2.8__py3-none-any.whl

fusion_bench/taskpool/nyuv2_taskpool.py

@@ -60,4 +60,6 @@ class NYUv2TaskPool(TaskPool):
             num_workers=self.config.num_workers,
         )
         report = self.trainer.validate(model, val_loader)
+        if isinstance(report, list) and len(report) == 1:
+            report = report[0]
         return report

fusion_bench/tasks/clip_classification/__init__.py

@@ -58,11 +58,24 @@ class CLIPTemplateFactory:
             "templates": "templates",
         },
         "nateraw/rendered-sst2": ".rendered_sst2",
+        "rendered-sst2": ".rendered_sst2",
         "tanganke/stl10": ".stl10",
+        "stl10": ".stl10",
         "dpdl-benchmark/oxford_flowers102": ".flower102",
+        "oxford_flowers102": ".flower102",
         "timm/oxford-iiit-pet": ".oxford_iiit_pet",
+        "oxford-iiit-pet": ".oxford_iiit_pet",
         "imagenet": ".imagenet",
         "tiny-imagenet": ".tiny_imagenet",
+        "pcam": ".pcam",
+        "fer2013": ".fer2013",
+        "emnist_mnist": ".emnist_mnist",
+        "emnist_letters": ".emnist_letters",
+        "kmnist": ".kmnist",
+        "food101": ".food101",
+        "fashion_mnist": ".fashion_mnist",
+        "cub-200-2011": ".cub_200_2011",
+        "mango-leaf-disease": ".mango_leaf_disease",
     }
 
     @staticmethod
@@ -168,48 +181,3 @@ class CLIPTemplateFactory:
 
 def get_classnames_and_templates(dataset_name: str):
     return CLIPTemplateFactory.get_classnames_and_templates(dataset_name)
-
-
-def _load_hf_dataset(dataset_name: str):
-    """
-    Load a dataset from the Hugging Face datasets library based on the specified dataset name.
-
-    This function handles specific preprocessing steps for certain datasets to ensure consistency in dataset format.
-    For example, it renames columns, removes unnecessary columns, and specifies subsets for certain datasets.
-
-    Expected dataset format:
-        - The dataset should have an "image" column containing the image data.
-        - The dataset should have a "label" column containing the class labels.
-
-    Args:
-        dataset_name (str): The name of the dataset to load. Can be one of "svhn", "cifar10", "cifar100", "timm/oxford-iiit-pet", or any other dataset name supported by the Hugging Face datasets library. By default, the datasets have two columns: "image" and "label".
-
-    Returns:
-        A dataset object loaded from the Hugging Face datasets library, with any necessary preprocessing applied.
-    """
-    if dataset_name == "svhn":
-        return load_dataset(dataset_name, "cropped_digits")
-    elif dataset_name == "cifar10":
-        dataset = load_dataset(dataset_name)
-        dataset = dataset.rename_columns({"img": "image"})
-        return dataset
-    elif dataset_name == "cifar100":
-        dataset = load_dataset(dataset_name)
-        dataset = dataset.remove_columns(["coarse_label"]).rename_columns(
-            {"img": "image", "fine_label": "label"}
-        )
-        return dataset
-    elif dataset_name == "timm/oxford-iiit-pet":
-        dataset = load_dataset(dataset_name)
-        dataset = dataset.remove_columns(["image_id", "label_cat_dog"])
-        return dataset
-    else:
-        return load_dataset(dataset_name)
-
-
-def load_clip_dataset(dataset: str, processor):
-    hf_dataset = _load_hf_dataset(dataset)
-    return (
-        CLIPDataset(hf_dataset["train"], processor),
-        CLIPDataset(hf_dataset["test"], processor),
-    )

fusion_bench/tasks/clip_classification/clip_dataset.py

@@ -1,16 +1 @@
-import torch
-
-
-class CLIPDataset(torch.utils.data.Dataset):
-    def __init__(self, dataset, processor):
-        self.dataset = dataset
-        self.processor = processor
-
-    def __len__(self):
-        return len(self.dataset)
-
-    def __getitem__(self, idx):
-        item = self.dataset[idx]
-        image = item["image"]
-        inputs = self.processor(images=[image], return_tensors="pt")["pixel_values"][0]
-        return inputs, item["label"]
+from fusion_bench.dataset.clip_dataset import CLIPDataset

fusion_bench/tasks/clip_classification/cub_200_2011.py

@@ -0,0 +1,208 @@
+classname_mapping = {
+    "0": "Black_footed_Albatross",
+    "1": "Laysan_Albatross",
+    "2": "Sooty_Albatross",
+    "3": "Groove_billed_Ani",
+    "4": "Crested_Auklet",
+    "5": "Least_Auklet",
+    "6": "Parakeet_Auklet",
+    "7": "Rhinoceros_Auklet",
+    "8": "Brewer_Blackbird",
+    "9": "Red_winged_Blackbird",
+    "10": "Rusty_Blackbird",
+    "11": "Yellow_headed_Blackbird",
+    "12": "Bobolink",
+    "13": "Indigo_Bunting",
+    "14": "Lazuli_Bunting",
+    "15": "Painted_Bunting",
+    "16": "Cardinal",
+    "17": "Spotted_Catbird",
+    "18": "Gray_Catbird",
+    "19": "Yellow_breasted_Chat",
+    "20": "Eastern_Towhee",
+    "21": "Chuck_will_Widow",
+    "22": "Brandt_Cormorant",
+    "23": "Red_faced_Cormorant",
+    "24": "Pelagic_Cormorant",
+    "25": "Bronzed_Cowbird",
+    "26": "Shiny_Cowbird",
+    "27": "Brown_Creeper",
+    "28": "American_Crow",
+    "29": "Fish_Crow",
+    "30": "Black_billed_Cuckoo",
+    "31": "Mangrove_Cuckoo",
+    "32": "Yellow_billed_Cuckoo",
+    "33": "Gray_crowned_Rosy_Finch",
+    "34": "Purple_Finch",
+    "35": "Northern_Flicker",
+    "36": "Acadian_Flycatcher",
+    "37": "Great_Crested_Flycatcher",
+    "38": "Least_Flycatcher",
+    "39": "Olive_sided_Flycatcher",
+    "40": "Scissor_tailed_Flycatcher",
+    "41": "Vermilion_Flycatcher",
+    "42": "Yellow_bellied_Flycatcher",
+    "43": "Frigatebird",
+    "44": "Northern_Fulmar",
+    "45": "Gadwall",
+    "46": "American_Goldfinch",
+    "47": "European_Goldfinch",
+    "48": "Boat_tailed_Grackle",
+    "49": "Eared_Grebe",
+    "50": "Horned_Grebe",
+    "51": "Pied_billed_Grebe",
+    "52": "Western_Grebe",
+    "53": "Blue_Grosbeak",
+    "54": "Evening_Grosbeak",
+    "55": "Pine_Grosbeak",
+    "56": "Rose_breasted_Grosbeak",
+    "57": "Pigeon_Guillemot",
+    "58": "California_Gull",
+    "59": "Glaucous_winged_Gull",
+    "60": "Heermann_Gull",
+    "61": "Herring_Gull",
+    "62": "Ivory_Gull",
+    "63": "Ring_billed_Gull",
+    "64": "Slaty_backed_Gull",
+    "65": "Western_Gull",
+    "66": "Anna_Hummingbird",
+    "67": "Ruby_throated_Hummingbird",
+    "68": "Rufous_Hummingbird",
+    "69": "Green_Violetear",
+    "70": "Long_tailed_Jaeger",
+    "71": "Pomarine_Jaeger",
+    "72": "Blue_Jay",
+    "73": "Florida_Jay",
+    "74": "Green_Jay",
+    "75": "Dark_eyed_Junco",
+    "76": "Tropical_Kingbird",
+    "77": "Gray_Kingbird",
+    "78": "Belted_Kingfisher",
+    "79": "Green_Kingfisher",
+    "80": "Pied_Kingfisher",
+    "81": "Ringed_Kingfisher",
+    "82": "White_breasted_Kingfisher",
+    "83": "Red_legged_Kittiwake",
+    "84": "Horned_Lark",
+    "85": "Pacific_Loon",
+    "86": "Mallard",
+    "87": "Western_Meadowlark",
+    "88": "Hooded_Merganser",
+    "89": "Red_breasted_Merganser",
+    "90": "Mockingbird",
+    "91": "Nighthawk",
+    "92": "Clark_Nutcracker",
+    "93": "White_breasted_Nuthatch",
+    "94": "Baltimore_Oriole",
+    "95": "Hooded_Oriole",
+    "96": "Orchard_Oriole",
+    "97": "Scott_Oriole",
+    "98": "Ovenbird",
+    "99": "Brown_Pelican",
+    "100": "White_Pelican",
+    "101": "Western_Wood_Pewee",
+    "102": "Sayornis",
+    "103": "American_Pipit",
+    "104": "Whip_poor_Will",
+    "105": "Horned_Puffin",
+    "106": "Common_Raven",
+    "107": "White_necked_Raven",
+    "108": "American_Redstart",
+    "109": "Geococcyx",
+    "110": "Loggerhead_Shrike",
+    "111": "Great_Grey_Shrike",
+    "112": "Baird_Sparrow",
+    "113": "Black_throated_Sparrow",
+    "114": "Brewer_Sparrow",
+    "115": "Chipping_Sparrow",
+    "116": "Clay_colored_Sparrow",
+    "117": "House_Sparrow",
+    "118": "Field_Sparrow",
+    "119": "Fox_Sparrow",
+    "120": "Grasshopper_Sparrow",
+    "121": "Harris_Sparrow",
+    "122": "Henslow_Sparrow",
+    "123": "Le_Conte_Sparrow",
+    "124": "Lincoln_Sparrow",
+    "125": "Nelson_Sharp_tailed_Sparrow",
+    "126": "Savannah_Sparrow",
+    "127": "Seaside_Sparrow",
+    "128": "Song_Sparrow",
+    "129": "Tree_Sparrow",
+    "130": "Vesper_Sparrow",
+    "131": "White_crowned_Sparrow",
+    "132": "White_throated_Sparrow",
+    "133": "Cape_Glossy_Starling",
+    "134": "Bank_Swallow",
+    "135": "Barn_Swallow",
+    "136": "Cliff_Swallow",
+    "137": "Tree_Swallow",
+    "138": "Scarlet_Tanager",
+    "139": "Summer_Tanager",
+    "140": "Artic_Tern",
+    "141": "Black_Tern",
+    "142": "Caspian_Tern",
+    "143": "Common_Tern",
+    "144": "Elegant_Tern",
+    "145": "Forsters_Tern",
+    "146": "Least_Tern",
+    "147": "Green_tailed_Towhee",
+    "148": "Brown_Thrasher",
+    "149": "Sage_Thrasher",
+    "150": "Black_capped_Vireo",
+    "151": "Blue_headed_Vireo",
+    "152": "Philadelphia_Vireo",
+    "153": "Red_eyed_Vireo",
+    "154": "Warbling_Vireo",
+    "155": "White_eyed_Vireo",
+    "156": "Yellow_throated_Vireo",
+    "157": "Bay_breasted_Warbler",
+    "158": "Black_and_white_Warbler",
+    "159": "Black_throated_Blue_Warbler",
+    "160": "Blue_winged_Warbler",
+    "161": "Canada_Warbler",
+    "162": "Cape_May_Warbler",
+    "163": "Cerulean_Warbler",
+    "164": "Chestnut_sided_Warbler",
+    "165": "Golden_winged_Warbler",
+    "166": "Hooded_Warbler",
+    "167": "Kentucky_Warbler",
+    "168": "Magnolia_Warbler",
+    "169": "Mourning_Warbler",
+    "170": "Myrtle_Warbler",
+    "171": "Nashville_Warbler",
+    "172": "Orange_crowned_Warbler",
+    "173": "Palm_Warbler",
+    "174": "Pine_Warbler",
+    "175": "Prairie_Warbler",
+    "176": "Prothonotary_Warbler",
+    "177": "Swainson_Warbler",
+    "178": "Tennessee_Warbler",
+    "179": "Wilson_Warbler",
+    "180": "Worm_eating_Warbler",
+    "181": "Yellow_Warbler",
+    "182": "Northern_Waterthrush",
+    "183": "Louisiana_Waterthrush",
+    "184": "Bohemian_Waxwing",
+    "185": "Cedar_Waxwing",
+    "186": "American_Three_toed_Woodpecker",
+    "187": "Pileated_Woodpecker",
+    "188": "Red_bellied_Woodpecker",
+    "189": "Red_cockaded_Woodpecker",
+    "190": "Red_headed_Woodpecker",
+    "191": "Downy_Woodpecker",
+    "192": "Bewick_Wren",
+    "193": "Cactus_Wren",
+    "194": "Carolina_Wren",
+    "195": "House_Wren",
+    "196": "Marsh_Wren",
+    "197": "Rock_Wren",
+    "198": "Winter_Wren",
+    "199": "Common_Yellowthroat",
+}
+
+classnames = [classname_mapping[str(i)] for i in range(200)]
+templates = [
+    lambda c: f"a photo of a {c}.",
+    lambda c: f"a photo of the {c}.",
+]

fusion_bench/tasks/clip_classification/emnist_letters.py

@@ -0,0 +1,31 @@
+classnames_mapping = {
+    "0": "A",
+    "1": "B",
+    "2": "C",
+    "3": "D",
+    "4": "E",
+    "5": "F",
+    "6": "G",
+    "7": "H",
+    "8": "I",
+    "9": "J",
+    "10": "K",
+    "11": "L",
+    "12": "M",
+    "13": "N",
+    "14": "O",
+    "15": "P",
+    "16": "Q",
+    "17": "R",
+    "18": "S",
+    "19": "T",
+    "20": "U",
+    "21": "V",
+    "22": "W",
+    "23": "X",
+    "24": "Y",
+    "25": "Z",
+}
+
+classnames = [classnames_mapping[str(i)] for i in range(26)]
+templates = [lambda c: f'a photo of the digit character: "{c}".']

fusion_bench/tasks/clip_classification/emnist_mnist.py

@@ -0,0 +1,5 @@
+# https://huggingface.co/datasets/tanganke/emnist_mnist
+classnames = [str(i) for i in range(10)]
+templates = [
+    lambda c: f'a photo of the number: "{c}".',
+]

fusion_bench/tasks/clip_classification/fashion_mnist.py

@@ -0,0 +1,18 @@
+classname_mapping = {
+    "0": "T - shirt / top",
+    "1": "Trouser",
+    "2": "Pullover",
+    "3": "Dress",
+    "4": "Coat",
+    "5": "Sandal",
+    "6": "Shirt",
+    "7": "Sneaker",
+    "8": "Bag",
+    "9": "Ankle boot",
+}
+classnames = [classname_mapping[str(i)] for i in range(10)]
+
+templates = [
+    lambda c: f"a photo of a {c}.",
+    lambda c: f"a photo of the {c}.",
+]

fusion_bench/tasks/clip_classification/fer2013.py

@@ -0,0 +1,18 @@
+classnames = [
+    "angry",
+    "disgusted",
+    "fearful",
+    "happy",
+    "neutral",
+    "sad",
+    "surprised",
+]
+
+templates = [
+    lambda c: f"a photo of a {c} looking face.",
+    lambda c: f"a photo of a face showing the emotion: {c}.",
+    lambda c: f"a photo of a face looking {c}.",
+    lambda c: f"a face that looks {c}.",
+    lambda c: f"they look {c}.",
+    lambda c: f"look at how {c} they are.",
+]

fusion_bench/tasks/clip_classification/food101.py

@@ -0,0 +1,105 @@
+classnames = [
+    "apple pie",
+    "baby back ribs",
+    "baklava",
+    "beef carpaccio",
+    "beef tartare",
+    "beet salad",
+    "beignets",
+    "bibimbap",
+    "bread pudding",
+    "breakfast burrito",
+    "bruschetta",
+    "caesar salad",
+    "cannoli",
+    "caprese salad",
+    "carrot cake",
+    "ceviche",
+    "cheese plate",
+    "cheesecake",
+    "chicken curry",
+    "chicken quesadilla",
+    "chicken wings",
+    "chocolate cake",
+    "chocolate mousse",
+    "churros",
+    "clam chowder",
+    "club sandwich",
+    "crab cakes",
+    "creme brulee",
+    "croque madame",
+    "cup cakes",
+    "deviled eggs",
+    "donuts",
+    "dumplings",
+    "edamame",
+    "eggs benedict",
+    "escargots",
+    "falafel",
+    "filet mignon",
+    "fish and chips",
+    "foie gras",
+    "french fries",
+    "french onion soup",
+    "french toast",
+    "fried calamari",
+    "fried rice",
+    "frozen yogurt",
+    "garlic bread",
+    "gnocchi",
+    "greek salad",
+    "grilled cheese sandwich",
+    "grilled salmon",
+    "guacamole",
+    "gyoza",
+    "hamburger",
+    "hot and sour soup",
+    "hot dog",
+    "huevos rancheros",
+    "hummus",
+    "ice cream",
+    "lasagna",
+    "lobster bisque",
+    "lobster roll sandwich",
+    "macaroni and cheese",
+    "macarons",
+    "miso soup",
+    "mussels",
+    "nachos",
+    "omelette",
+    "onion rings",
+    "oysters",
+    "pad thai",
+    "paella",
+    "pancakes",
+    "panna cotta",
+    "peking duck",
+    "pho",
+    "pizza",
+    "pork chop",
+    "poutine",
+    "prime rib",
+    "pulled pork sandwich",
+    "ramen",
+    "ravioli",
+    "red velvet cake",
+    "risotto",
+    "samosa",
+    "sashimi",
+    "scallops",
+    "seaweed salad",
+    "shrimp and grits",
+    "spaghetti bolognese",
+    "spaghetti carbonara",
+    "spring rolls",
+    "steak",
+    "strawberry shortcake",
+    "sushi",
+    "tacos",
+    "takoyaki",
+    "tiramisu",
+    "tuna tartare",
+    "waffles",
+]
+
+templates = [lambda c: f"a photo of {c}, a type of food."]

fusion_bench/tasks/clip_classification/kmnist.py

@@ -0,0 +1,17 @@
+classnames_mapping = {
+    "0": "お",
+    "1": "き",
+    "2": "す",
+    "3": "つ",
+    "4": "な",
+    "5": "は",
+    "6": "ま",
+    "7": "や",
+    "8": "れ",
+    "9": "を",
+}
+classnames = [classnames_mapping[str(c)] for c in range(10)]
+
+templates = [
+    lambda c: f"a photo of the character {c}.",
+]

fusion_bench/tasks/clip_classification/mongo_leaf_disease.py

@@ -0,0 +1,19 @@
+classnames = [
+    "Anthracnose",
+    "Bacterial Canker",
+    "Cutting Weevil",
+    "Die Back",
+    "Gall Midge",
+    "Healthy",
+    "Powdery Mildew",
+    "Sooty Mould",
+]
+
+templates = [
+    lambda c: f"a photo of a mango leaf with {c}.",
+    lambda c: f"a mango leaf showing symptoms of {c}.",
+    lambda c: f"a close-up photo of {c} on a mango leaf.",
+    lambda c: f"this mango leaf is affected by {c}.",
+    lambda c: f"a mango leaf disease identified as {c}.",
+    lambda c: f"a {c} infection on a mango leaf.",
+]

fusion_bench/tasks/clip_classification/pcam.py

@@ -0,0 +1,5 @@
+classnames = ["lymph node", "lymph node containing metastatic tumor tissue"]
+
+templates = [
+    lambda c: f"this is a photo of {c}",
+]

fusion_bench/utils/hydra_utils.py

@@ -4,3 +4,25 @@ import hydra.core.hydra_config
 def get_hydra_output_dir():
     hydra_cfg = hydra.core.hydra_config.HydraConfig.get()
     return hydra_cfg.runtime.output_dir
+
+
+def config_priority_get(priority_config, general_config, key, default):
+    """
+    Retrieve a configuration value with priority.
+
+    This function retrieves the value associated with `key` from `priority_config` if it exists.
+    If the key is not found in `priority_config`, it retrieves the value from `general_config`.
+    If the key is not found in either configuration, it returns the provided `default` value.
+
+    Args:
+        priority_config (dict): The configuration dictionary with higher priority.
+        general_config (dict): The general configuration dictionary.
+        key (str): The key to look up in the configuration dictionaries.
+        default: The default value to return if the key is not found in either configuration.
+
+    Returns:
+        The value associated with `key` from `priority_config` or `general_config`, or the `default` value if the key is not found.
+    """
+    if key in priority_config:
+        return priority_config[key]
+    return general_config.get(key, default)

fusion_bench/utils/parameters.py

@@ -1,6 +1,6 @@
 import copy
 from collections import OrderedDict
-from typing import List, Mapping, Union
+from typing import List, Mapping, Optional, Union
 
 import torch
 from torch import nn
@@ -43,7 +43,10 @@ def trainable_state_dict(
     return state_dict
 
 
-def state_dict_to_vector(state_dict, remove_keys=[]):
+def state_dict_to_vector(
+    state_dict: StateDictType,
+    remove_keys: Optional[List[str]] = None,
+):
     """
     Convert a state dictionary to a vector.
 
@@ -54,6 +57,7 @@ def state_dict_to_vector(state_dict, remove_keys=[]):
     Returns:
         torch.Tensor: The converted vector.
     """
+    remove_keys = remove_keys if remove_keys is not None else []
     shared_state_dict = copy.deepcopy(state_dict)
     for key in remove_keys:
         if key in shared_state_dict:
@@ -64,7 +68,11 @@ def state_dict_to_vector(state_dict, remove_keys=[]):
     )
 
 
-def vector_to_state_dict(vector, state_dict, remove_keys=[]):
+def vector_to_state_dict(
+    vector: torch.Tensor,
+    state_dict: StateDictType,
+    remove_keys: Optional[List[str]] = None,
+):
     """
     Convert a vector to a state dictionary.
 
@@ -76,6 +84,7 @@ def vector_to_state_dict(vector, state_dict, remove_keys=[]):
     Returns:
         dict: The converted state dictionary.
     """
+    remove_keys = remove_keys if remove_keys is not None else []
     # create a reference dict to define the order of the vector
     reference_dict = copy.deepcopy(state_dict)
     for key in remove_keys:

fusion_bench/utils/plot/token.py

@@ -0,0 +1,52 @@
+import matplotlib.pyplot as plt
+import numpy as np
+import seaborn as sns
+
+
+def visualize_model_inputs(input_ids, attention_mask, labels, tokenizer=None):
+    """
+    Visualize model inputs: attention mask, labels and input_ids
+
+    Parameters:
+    -----------
+    attention_mask: numpy array or tensor
+        The attention mask array
+    labels: numpy array or tensor
+        The labels array
+    input_ids: numpy array or tensor
+        The input ids array
+    tokenizer: optional
+        The tokenizer object to decode input_ids
+    """
+
+    # Convert inputs to numpy if they're tensors
+    attention_mask = np.array(attention_mask)
+    labels = np.array(labels)
+    input_ids = np.array(input_ids)
+
+    # Create figure with 3 subplots
+    fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(15, 10))
+
+    # Plot attention mask
+    sns.heatmap(attention_mask.reshape(1, -1), ax=ax1, cmap="Blues", cbar=True)
+    ax1.set_title("**Attention Mask**")
+    ax1.set_ylabel("Sequence")
+
+    # Plot labels
+    sns.heatmap(labels.reshape(1, -1), ax=ax2, cmap="Reds", cbar=True)
+    ax2.set_title("**Labels**")
+    ax2.set_ylabel("Sequence")
+
+    # Plot input_ids
+    sns.heatmap(input_ids.reshape(1, -1), ax=ax3, cmap="Greens", cbar=True)
+    ax3.set_title("**Input IDs**")
+    ax3.set_ylabel("Sequence")
+
+    # If tokenizer is provided, add decoded tokens as x-axis labels
+    if tokenizer:
+        decoded_tokens = [tokenizer.decode(token_id) for token_id in input_ids]
+        ax3.set_xticks(np.arange(len(decoded_tokens)) + 0.5)
+        ax3.set_xticklabels(decoded_tokens, rotation=45, ha="right")
+
+    plt.tight_layout()
+    return fig