crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/scenarios/audio_language/fleurs_scenario.py

@@ -0,0 +1,312 @@
+"""Scenarios for audio models"""
+
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from collections import OrderedDict
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+
+
+class FLEURSScenario(Scenario):
+    """FLEURS Scenario
+
+    The FLEURS (Conneau et al, 2022) dataset is an n-way parallel speech dataset in 102 languages
+    built on top of the machine translation FLoRes-101 benchmark, with approximately 12 hours of speech
+    supervision per language. The task is to identify the language used from the audio sample
+    (the Speech Language Identification task).
+
+    Paper: https://arxiv.org/abs/2205.12446
+    Code: https://tensorflow.org/datasets/catalog/xtreme_s
+
+    Citation:
+    @inproceedings{conneau2023fleurs,
+        title={Fleurs: Few-shot learning evaluation of universal representations of speech},
+        author={Conneau, Alexis and Ma, Min and Khanuja, Simran and Zhang, Yu and Axelrod,
+        Vera and Dalmia, Siddharth and Riesa, Jason and Rivera, Clara and Bapna, Ankur},
+        booktitle={2022 IEEE Spoken Language Technology Workshop (SLT)},
+        pages={798--805},
+        year={2023},
+        organization={IEEE}
+        }
+    """
+
+    HF_DATASET_NAME = "google/xtreme_s"
+
+    _FLEURS_LANG_TO_ID = OrderedDict(
+        [
+            ("Afrikaans", "af"),
+            ("Amharic", "am"),
+            ("Arabic", "ar"),
+            ("Armenian", "hy"),
+            ("Assamese", "as"),
+            ("Asturian", "ast"),
+            ("Azerbaijani", "az"),
+            ("Belarusian", "be"),
+            ("Bengali", "bn"),
+            ("Bosnian", "bs"),
+            ("Bulgarian", "bg"),
+            ("Burmese", "my"),
+            ("Catalan", "ca"),
+            ("Cebuano", "ceb"),
+            ("Mandarin_Chinese", "cmn_hans"),
+            ("Cantonese_Chinese", "yue_hant"),
+            ("Croatian", "hr"),
+            ("Czech", "cs"),
+            ("Danish", "da"),
+            ("Dutch", "nl"),
+            ("English", "en"),
+            ("Estonian", "et"),
+            ("Filipino", "fil"),
+            ("Finnish", "fi"),
+            ("French", "fr"),
+            ("Fula", "ff"),
+            ("Galician", "gl"),
+            ("Ganda", "lg"),
+            ("Georgian", "ka"),
+            ("German", "de"),
+            ("Greek", "el"),
+            ("Gujarati", "gu"),
+            ("Hausa", "ha"),
+            ("Hebrew", "he"),
+            ("Hindi", "hi"),
+            ("Hungarian", "hu"),
+            ("Icelandic", "is"),
+            ("Igbo", "ig"),
+            ("Indonesian", "id"),
+            ("Irish", "ga"),
+            ("Italian", "it"),
+            ("Japanese", "ja"),
+            ("Javanese", "jv"),
+            ("Kabuverdianu", "kea"),
+            ("Kamba", "kam"),
+            ("Kannada", "kn"),
+            ("Kazakh", "kk"),
+            ("Khmer", "km"),
+            ("Korean", "ko"),
+            ("Kyrgyz", "ky"),
+            ("Lao", "lo"),
+            ("Latvian", "lv"),
+            ("Lingala", "ln"),
+            ("Lithuanian", "lt"),
+            ("Luo", "luo"),
+            ("Luxembourgish", "lb"),
+            ("Macedonian", "mk"),
+            ("Malay", "ms"),
+            ("Malayalam", "ml"),
+            ("Maltese", "mt"),
+            ("Maori", "mi"),
+            ("Marathi", "mr"),
+            ("Mongolian", "mn"),
+            ("Nepali", "ne"),
+            ("Northern-Sotho", "nso"),
+            ("Norwegian", "nb"),
+            ("Nyanja", "ny"),
+            ("Occitan", "oc"),
+            ("Oriya", "or"),
+            ("Oromo", "om"),
+            ("Pashto", "ps"),
+            ("Persian", "fa"),
+            ("Polish", "pl"),
+            ("Portuguese", "pt"),
+            ("Punjabi", "pa"),
+            ("Romanian", "ro"),
+            ("Russian", "ru"),
+            ("Serbian", "sr"),
+            ("Shona", "sn"),
+            ("Sindhi", "sd"),
+            ("Slovak", "sk"),
+            ("Slovenian", "sl"),
+            ("Somali", "so"),
+            ("Sorani-Kurdish", "ckb"),
+            ("Spanish", "es"),
+            ("Swahili", "sw"),
+            ("Swedish", "sv"),
+            ("Tajik", "tg"),
+            ("Tamil", "ta"),
+            ("Telugu", "te"),
+            ("Thai", "th"),
+            ("Turkish", "tr"),
+            ("Ukrainian", "uk"),
+            ("Umbundu", "umb"),
+            ("Urdu", "ur"),
+            ("Uzbek", "uz"),
+            ("Vietnamese", "vi"),
+            ("Welsh", "cy"),
+            ("Wolof", "wo"),
+            ("Xhosa", "xh"),
+            ("Yoruba", "yo"),
+            ("Zulu", "zu"),
+        ]
+    )
+    _FLEURS_LANG = sorted(
+        [
+            "af_za",
+            "am_et",
+            "ar_eg",
+            "as_in",
+            "ast_es",
+            "az_az",
+            "be_by",
+            "bn_in",
+            "bs_ba",
+            "ca_es",
+            "ceb_ph",
+            "cmn_hans_cn",
+            "yue_hant_hk",
+            "cs_cz",
+            "cy_gb",
+            "da_dk",
+            "de_de",
+            "el_gr",
+            "en_us",
+            "es_419",
+            "et_ee",
+            "fa_ir",
+            "ff_sn",
+            "fi_fi",
+            "fil_ph",
+            "fr_fr",
+            "ga_ie",
+            "gl_es",
+            "gu_in",
+            "ha_ng",
+            "he_il",
+            "hi_in",
+            "hr_hr",
+            "hu_hu",
+            "hy_am",
+            "id_id",
+            "ig_ng",
+            "is_is",
+            "it_it",
+            "ja_jp",
+            "jv_id",
+            "ka_ge",
+            "kam_ke",
+            "kea_cv",
+            "kk_kz",
+            "km_kh",
+            "kn_in",
+            "ko_kr",
+            "ckb_iq",
+            "ky_kg",
+            "lb_lu",
+            "lg_ug",
+            "ln_cd",
+            "lo_la",
+            "lt_lt",
+            "luo_ke",
+            "lv_lv",
+            "mi_nz",
+            "mk_mk",
+            "ml_in",
+            "mn_mn",
+            "mr_in",
+            "ms_my",
+            "mt_mt",
+            "my_mm",
+            "nb_no",
+            "ne_np",
+            "nl_nl",
+            "nso_za",
+            "ny_mw",
+            "oc_fr",
+            "om_et",
+            "or_in",
+            "pa_in",
+            "pl_pl",
+            "ps_af",
+            "pt_br",
+            "ro_ro",
+            "ru_ru",
+            "bg_bg",
+            "sd_in",
+            "sk_sk",
+            "sl_si",
+            "sn_zw",
+            "so_so",
+            "sr_rs",
+            "sv_se",
+            "sw_ke",
+            "ta_in",
+            "te_in",
+            "tg_tj",
+            "th_th",
+            "tr_tr",
+            "uk_ua",
+            "umb_ao",
+            "ur_pk",
+            "uz_uz",
+            "vi_vn",
+            "wo_sn",
+            "xh_za",
+            "yo_ng",
+            "zu_za",
+        ]
+    )
+
+    # Randomly selected 7 languages from 7 different groups (western_european_we, eastern_european_ee,
+    # central_asia_middle_north_african_cmn, sub_saharan_african_ssa, south_asian_sa, south_east_asian_sea,
+    # chinese_japanase_korean_cjk) in the FLEURS dataset.
+    _FLEURS_TEST_LANG_TO_ID = OrderedDict(
+        [
+            ("Finnish", "fi"),
+            ("English", "en"),
+            ("Hebrew", "he"),
+            ("Zulu", "zu"),
+            ("Bengali", "bn"),
+            ("Thai", "th"),
+            ("Mandarin_Chinese", "cmn_hans"),
+        ]
+    )
+
+    name = "fleurs"
+    description = "Language identification for seven languages from seven different language groups \
+        ([Conneau et al, 2022](https://arxiv.org/abs/2205.12446))."
+    tags: List[str] = ["audio", "recognition", "multilinguality"]
+
+    def __init__(self, language: str) -> None:
+        super().__init__()
+
+        if language not in FLEURSScenario._FLEURS_TEST_LANG_TO_ID.keys():
+            raise ValueError(
+                f"Invalid language: {language}. Valid languages are: {FLEURSScenario._FLEURS_TEST_LANG_TO_ID.keys()}"
+            )
+
+        self._fleurs_lang_short_to_long = {v: k for k, v in FLEURSScenario._FLEURS_LANG_TO_ID.items()}
+        self._fleurs_long_to_lang = {
+            self._fleurs_lang_short_to_long["_".join(k.split("_")[:-1]) or k]: k for k in FLEURSScenario._FLEURS_LANG
+        }
+
+        self._language: str = language
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        language_category = self._fleurs_long_to_lang[self._language]
+        for row in tqdm(
+            load_dataset(
+                FLEURSScenario.HF_DATASET_NAME,
+                name=f"fleurs.{language_category}",
+                cache_dir=output_path,
+                split=TEST_SPLIT,
+                trust_remote_code=True,
+            )
+        ):
+            local_audio_path = row["path"]
+            answer = row["transcription"]
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/wav", location=local_audio_path)])
+            )
+            references = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/iemocap_audio_scenario.py

@@ -0,0 +1,83 @@
+import datasets
+import os
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+from helm.common.general import ensure_directory_exists
+from helm.common.media_object import MediaObject, MultimediaObject
+
+
+class IEMOCAPAudioScenario(Scenario):
+    """IEMOCAP (Audio)
+
+    This scenario is a emotion classification scenario based on the
+    "Interactive emotional dyadic motion capture database" (IEMOCAP),
+    collected by the Speech Analysis and Interpretation Laboratory (SAIL)
+    at the University of Southern California (USC). Only the audio data
+    from this dataset is used. The task is to classify the emotion of the
+    speaker(s) in the audio sample as one of angry, happy, neutral or sad.
+
+    Website: https://sail.usc.edu/iemocap/iemocap_release.htm
+    Paper: https://sail.usc.edu/iemocap/Busso_2008_iemocap.pdf
+    Dataset: https://huggingface.co/datasets/Zahra99/IEMOCAP_Audio/blob/main/README.md
+
+    Citation:
+    @article{article,
+    author = {Busso, Carlos and Bulut, Murtaza and Lee, Chi-Chun and Kazemzadeh, Abe and Mower Provost, Emily and Kim, Samuel and Chang, Jeannette and Lee, Sungbok and Narayanan, Shrikanth},
+    year = {2008},
+    month = {12},
+    pages = {335-359},
+    title = {IEMOCAP: Interactive emotional dyadic motion capture database},
+    volume = {42},
+    journal = {Language Resources and Evaluation},
+    doi = {10.1007/s10579-008-9076-6}
+    }
+    """  # noqa: E501
+
+    name = "iemocap_audio"
+    description = "A classification scenario based on audio data from the Interactive emotional dyadic motion capture database (IEMOCAP) ([Busso et al, 2008](https://sail.usc.edu/iemocap/Busso_2008_iemocap.pdf)). The task is to classify the emotion of the speaker(s) in the audio sample."  # noqa: E501
+    tags = ["audio", "classification"]
+
+    LABEL_NAMES = ["angry", "happy", "neutral", "sad"]
+    SAMPLE_RATE = 16000
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        wav_dir = os.path.join(output_path, "wav")
+        ensure_directory_exists(wav_dir)
+
+        dataset = datasets.load_dataset(
+            "Zahra99/IEMOCAP_Audio", revision="4f8539a397ecc0d7185bf941bc1bb7238abc3648", cache_dir=cache_dir
+        )
+        instances: List[Instance] = []
+        for _, split in dataset.items():
+            for row in split:
+                wav_path = os.path.join(wav_dir, row["audio"]["path"])
+                print(len(row["audio"]["array"]))
+                print(list(row["audio"]["array"])[0:10])
+                ensure_audio_file_exists_from_array(
+                    wav_path, row["audio"]["array"], sample_rate=IEMOCAPAudioScenario.SAMPLE_RATE
+                )
+                input = Input(
+                    multimedia_content=MultimediaObject(
+                        media_objects=[MediaObject(location=wav_path, content_type="audio/wav")]
+                    )
+                )
+                references = [
+                    Reference(output=Output(text=IEMOCAPAudioScenario.LABEL_NAMES[row["label"]]), tags=[CORRECT_TAG])
+                ]
+                instance = Instance(input=input, references=references, split=TEST_SPLIT)
+                instances.append(instance)
+                print(row["audio"])
+                break
+        return instances

helm/benchmark/scenarios/audio_language/librispeech_fairness_scenario.py

@@ -0,0 +1,96 @@
+from typing import List
+import os
+import json
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from collections import OrderedDict
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+from helm.common.general import ensure_file_downloaded
+
+
+class LibriSpeechFairnessScenario(Scenario):
+    """LibriSpeech Fairness Scenario
+    The LibriSpeech corpus (Vassil et al. 2015) is derived from audiobooks that are part of the LibriVox
+    project, and contains 1000 hours of speech sampled at 16 kHz. The data has separately prepared language-model
+    training data and pre-built language models. This corpus is one of the most widely-used ASR corpus, which
+    has been extended to many applicaitons such as robust ASR and multilingual ASR tasks.
+
+    Paper: https://ieeexplore.ieee.org/document/7178964
+    Code: https://www.openslr.org/12
+
+    Citation:
+    @INPROCEEDINGS{7178964,
+        author={Panayotov, Vassil and Chen, Guoguo and Povey, Daniel and Khudanpur, Sanjeev},
+        booktitle={2015 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+        title={Librispeech: An ASR corpus based on public domain audio books},
+        year={2015},
+        doi={10.1109/ICASSP.2015.7178964}}
+    """
+
+    HF_DATASET_NAME = "openslr/librispeech_asr"
+    HF_MAPPING_URL = (
+        "https://huggingface.co/datasets/PahaII/SRB_instance_key_mapping/resolve/main/srb_instance_keys.json"
+    )
+    GENDER_MAPPING_URL = (
+        "https://huggingface.co/datasets/PahaII/librispeech_id2gender/resolve/main/librispeech_id2gender.json"
+    )
+    GENDERS = ["male", "female"]
+
+    name = "librispeech_fairness"
+    description = (
+        "Widely-used speech corpus for the speech recognition task "
+        "([Vassil et al. 2015](https://ieeexplore.ieee.org/document/7178964))."
+    )
+    tags: List[str] = ["audio", "recognition"]
+
+    def __init__(self, gender: str) -> None:
+        super().__init__()
+
+        if gender.lower() not in LibriSpeechFairnessScenario.GENDERS:
+            raise ValueError(
+                f"Invalid gender input: {gender}. Valid languages are: {LibriSpeechFairnessScenario.GENDERS}"
+            )
+
+        self._gender: str = gender
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        mapping_local_path = os.path.join(output_path, "librispeech_id2gender.json")
+        ensure_file_downloaded(
+            source_url=LibriSpeechFairnessScenario.GENDER_MAPPING_URL, target_path=mapping_local_path
+        )
+        meta_data = load_dataset(
+            LibriSpeechFairnessScenario.HF_DATASET_NAME,
+            name="clean",
+            cache_dir=output_path,
+            split=TEST_SPLIT,
+        )
+        gender_mapping_dict = json.load(open(mapping_local_path))
+        loading_cases: List[OrderedDict] = []
+        for row in tqdm(meta_data):
+            if gender_mapping_dict[str(row["speaker_id"])] == self._gender.lower():
+                loading_cases.append(row)
+
+        for row in tqdm(loading_cases):
+            local_audio_name = f"librispeech_{row['id']}.mp3"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+            answer = row["text"].lower()
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/mp3", location=local_audio_path)])
+            )
+            references = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/librispeech_scenario.py

@@ -0,0 +1,80 @@
+from typing import List
+import os
+import json
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+from helm.common.general import ensure_file_downloaded
+
+
+class LibriSpeechScenario(Scenario):
+    """LibriSpeech Corpus
+    The LibriSpeech corpus (Vassil et al. 2015) is derived from audiobooks that are part of the LibriVox
+    project, and contains 1000 hours of speech sampled at 16 kHz. The data has separately prepared language-model
+    training data and pre-built language models. This corpus is one of the most widely-used ASR corpus, which
+    has been extended to many applicaitons such as robust ASR and multilingual ASR tasks.
+
+    Paper: https://ieeexplore.ieee.org/document/7178964
+    Code: https://www.openslr.org/12
+
+    Citation:
+    @INPROCEEDINGS{7178964,
+        author={Panayotov, Vassil and Chen, Guoguo and Povey, Daniel and Khudanpur, Sanjeev},
+        booktitle={2015 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+        title={Librispeech: An ASR corpus based on public domain audio books},
+        year={2015},
+        doi={10.1109/ICASSP.2015.7178964}}
+    """
+
+    HF_DATASET_NAME = "openslr/librispeech_asr"
+    HF_MAPPING_URL = (
+        "https://huggingface.co/datasets/PahaII/SRB_instance_key_mapping/resolve/main/srb_instance_keys.json"
+    )
+    SRB_KEY = "srb_librispeech_noises_key2audio"
+    SRB_SUBSET = "gnoise.1"
+    MAPPING_KEY = "librispeech_id2line"
+
+    name = "librispeech"
+    description = (
+        "Widely-used speech corpus for the speech recognition task "
+        "([Vassil et al. 2015](https://ieeexplore.ieee.org/document/7178964))."
+    )
+    tags: List[str] = ["audio", "recognition"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        mapping_local_path = os.path.join(output_path, "srb_instance_keys.json")
+        ensure_file_downloaded(source_url=LibriSpeechScenario.HF_MAPPING_URL, target_path=mapping_local_path)
+        meta_data = load_dataset(
+            LibriSpeechScenario.HF_DATASET_NAME,
+            name="clean",
+            cache_dir=output_path,
+            split=TEST_SPLIT,
+        )
+        mapping_dict = json.load(open(mapping_local_path))
+        srb_mapping_keys = mapping_dict[self.SRB_KEY][self.SRB_SUBSET]
+        index2line_num = mapping_dict[self.MAPPING_KEY]
+        for line_num in tqdm(list(srb_mapping_keys)):
+            row = meta_data[int(index2line_num[line_num])]
+            local_audio_name = f"{self.name}_{line_num}.mp3"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+            answer = row["text"].lower()
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/mp3", location=local_audio_path)])
+            )
+            references = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/meld_audio_scenario.py

@@ -0,0 +1,113 @@
+import os
+from typing import List
+
+import pandas as pd
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.audio_utils import ensure_audio_file_exists_from_array, get_array_from_audio_file
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class MELDAudioScenario(Scenario):
+    """Multimodal EmotionLines Dataset (MELD) Audio
+
+    Multimodal EmotionLines Dataset (MELD) has been created by enhancing and extending EmotionLines dataset.
+    MELD has more than 1400 dialogues and 13000 utterances from Friends TV series. Multiple speakers participated
+    in the dialogues. Each utterance in a dialogue has been labeled by any of these seven emotions -
+    Anger, Disgust, Sadness, Joy, Neutral, Surprise and Fear.
+
+    The task is to classify the emotion based on only the audio clip.
+
+    Website: https://affective-meld.github.io/
+    Paper: https://arxiv.org/abs/1810.02508
+    Dataset: https://huggingface.co/datasets/DavidCombei/Wav2Vec_MELD_Audio
+
+    Citation:
+    S. Poria, D. Hazarika, N. Majumder, G. Naik, R. Mihalcea,
+    E. Cambria. MELD: A Multimodal Multi-Party Dataset
+    for Emotion Recognition in Conversation. (2018)
+
+    Chen, S.Y., Hsu, C.C., Kuo, C.C. and Ku, L.W.
+    EmotionLines: An Emotion Corpus of Multi-Party
+    Conversations. arXiv preprint arXiv:1802.08379 (2018).
+    """  # noqa: E501
+
+    name = "meld_audio"
+    description = "Classify emotions in audio clips from the television series Friends ([Poria et al, 2018](https://arxiv.org/abs/1810.02508))."  # noqa: E501
+    tags = ["audio", "classification"]
+
+    LABEL_NAMES = ["anger", "disgust", "fear", "joy", "neutral", "sadness", "surprise"]
+    CSV_URL_PREFIX = (
+        "https://raw.githubusercontent.com/declare-lab/MELD/2d2011b409d3ca2d7e94460cd007d434b1d0a102/data/MELD/"
+    )
+    SPLIT_NAME_TO_CSV_FILE_NAME = {
+        VALID_SPLIT: "dev_sent_emo.csv",
+        TRAIN_SPLIT: "train_sent_emo.csv",
+        TEST_SPLIT: "test_sent_emo.csv",
+    }
+    SPLIT_NAME_TO_TGZ_FILE_NAME = {
+        VALID_SPLIT: "audios_validation.tgz",
+        TRAIN_SPLIT: "audios_train.tgz",
+        TEST_SPLIT: "audios_test.tgz",
+    }
+    SAMPLE_RATE = 16000
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        csv_dir = os.path.join(output_path, "csv")
+        ensure_directory_exists(csv_dir)
+
+        instances: List[Instance] = []
+        # Only download the test split.
+        # We don't need the train split (because we use zero shot) or the validation split.
+        split_name = TEST_SPLIT
+
+        # Download the CSV to get the labels and IDs
+        csv_file_name = MELDAudioScenario.SPLIT_NAME_TO_CSV_FILE_NAME[split_name]
+        csv_file_path = os.path.join(csv_dir, csv_file_name)
+        ensure_file_downloaded(MELDAudioScenario.CSV_URL_PREFIX + csv_file_name, csv_file_path)
+        df = pd.read_csv(csv_file_path, header=0).rename(columns={"Sr No.": "serial_number"})
+
+        # Download FLAC files
+        flac_dir = os.path.join(output_path, f"flac_{split_name}")
+        ensure_file_downloaded(
+            source_url=f"https://huggingface.co/datasets/zrr1999/MELD_Text_Audio/resolve/main/archive/{MELDAudioScenario.SPLIT_NAME_TO_TGZ_FILE_NAME[split_name]}?download=true",  # noqa: E501
+            target_path=flac_dir,
+            unpack=True,
+            unpack_type="untar",
+        )
+
+        wav_dir = os.path.join(output_path, f"wav_{split_name}")
+        ensure_directory_exists(wav_dir)
+        for row in tqdm(df.itertuples()):
+            # Transcode FLAC to WAV
+            wav_file_name = f"dia{row.Dialogue_ID}_utt{row.Utterance_ID}.wav"
+            wav_file_path = os.path.join(wav_dir, wav_file_name)
+            if not os.path.isfile(wav_file_path):
+                flac_file_name = f"dia{row.Dialogue_ID}_utt{row.Utterance_ID}.flac"
+                flac_file_path = os.path.join(flac_dir, flac_file_name)
+                audio_array = get_array_from_audio_file(flac_file_path, MELDAudioScenario.SAMPLE_RATE)
+                ensure_audio_file_exists_from_array(wav_file_path, audio_array, MELDAudioScenario.SAMPLE_RATE)
+            input = Input(
+                multimedia_content=MultimediaObject(
+                    media_objects=[MediaObject(location=wav_file_path, content_type="audio/wav")]
+                )
+            )
+            assert row.Emotion in MELDAudioScenario.LABEL_NAMES
+            references = [Reference(output=Output(text=row.Emotion), tags=[CORRECT_TAG])]
+            instance = Instance(
+                id=str(f"awoo{row.serial_number}"), input=input, references=references, split=split_name
+            )
+            instances.append(instance)
+        return instances