crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/scenarios/audio_language/air_bench_foundation_scenario.py

@@ -0,0 +1,154 @@
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+from helm.common.audio_utils import is_invalid_audio_file
+import json
+
+
+class AirBenchFoundationScenario(Scenario):
+    """Air-Bench Foundation
+
+    Air-Bench AIR-Bench (Audio InstRuction Benchmark) is a benchmark designed to evaluate the ability of audio language
+    models to understand various types of audio signals (including human speech, natural sounds and music), and
+    furthermore, to interact with humans in textual format. AIR-Bench encompasses two dimensions: foundation
+    and chat benchmarks. The former consists of 19 tasks with approximately 19k single-choice questions. The
+    latter one contains 2k instances of open-ended question-and-answer data. We consider the chat benchmark
+    in this scenario.
+
+    Paper: https://aclanthology.org/2024.acl-long.109.pdf
+    Code: https://github.com/OFA-Sys/AIR-Bench
+
+    Citation:
+    @inproceedings{yang-etal-2024-air,
+    title = "{AIR}-Bench: Benchmarking Large Audio-Language Models via Generative Comprehension",
+    author = "Yang, Qian  and
+      Xu, Jin  and
+      Liu, Wenrui  and
+      Chu, Yunfei  and
+      Jiang, Ziyue  and
+      Zhou, Xiaohuan  and
+      Leng, Yichong  and
+      Lv, Yuanjun  and
+      Zhao, Zhou  and
+      Zhou, Chang  and
+      Zhou, Jingren",
+    booktitle = "Proceedings of the 62nd Annual Meeting of the Association for Computational
+        Linguistics (Volume 1: Long Papers)",
+    year = "2024",}
+    """
+
+    HF_DATA_PATH_PREFIX = "https://huggingface.co/datasets/qyang1021/AIR-Bench-Dataset/resolve/main/Foundation"
+    META_DATA_FILE_PATH = (
+        "https://huggingface.co/datasets/qyang1021/AIR-Bench-Dataset/resolve/main/Foundation/Foundation_meta.json"
+    )
+    SUBJECTS_DICT = {
+        "music_instrument_classification": "Music_Instruments_Classfication",
+        "music_genre_recognition": "Music_Genre_Recognition",
+        "music_qa": "Music_AQA",
+    }
+    OPTION_KEYS = ["choice_a", "choice_b", "choice_c", "choice_d"]
+
+    name = "air_bench_foundation"
+    description = "A large-scale dataset of about 46K audio clips to human-written text pairs \
+        ([Yang et al, 2024](https://aclanthology.org/2024.acl-long.109.pdf))."
+    tags: List[str] = ["audio", "classification", "knowledge"]
+
+    def __init__(self, subject: str) -> None:
+        super().__init__()
+
+        if subject not in AirBenchFoundationScenario.SUBJECTS_DICT.keys():
+            raise ValueError(f"Invalid subject. Valid subjects are: {AirBenchFoundationScenario.SUBJECTS_DICT.keys()}")
+
+        self._subject: str = subject
+
+    def _get_subject_indices(self, meta_data) -> List[int]:
+        subject_indices = []
+        for idx, line in enumerate(meta_data):
+            if line["task_name"] == self.SUBJECTS_DICT[self._subject]:
+                subject_indices.append(idx)
+        return subject_indices
+
+    def _get_content_type(self, audio_file_name) -> str:
+        if audio_file_name.endswith(".wav"):
+            return "audio/wav"
+        elif audio_file_name.endswith(".mp3"):
+            return "audio/mp3"
+        else:
+            raise ValueError(f"Unsupported audio file format: {audio_file_name}")
+
+    def _get_label_from_answer(self, row: dict, answer: str):
+        for option_key in self.OPTION_KEYS:
+            if row[option_key] == answer:
+                label = option_key.split("_")[-1].capitalize()
+                return label
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        data_dir: str = os.path.join(output_path, "audio_files")
+        meta_data_path: str = os.path.join(output_path, "Foundation_meta.json")
+        ensure_file_downloaded(source_url=AirBenchFoundationScenario.META_DATA_FILE_PATH, target_path=meta_data_path)
+        meta_data = json.load(open(meta_data_path))
+        subject_indices = self._get_subject_indices(meta_data)
+        valid_testing_indices = []
+        for _, row in enumerate(subject_indices):
+            audio_meda_data = meta_data[row]
+            hf_audio_file_path = os.path.join(
+                self.HF_DATA_PATH_PREFIX,
+                f'{audio_meda_data["task_name"]}_{audio_meda_data["dataset_name"]}/{audio_meda_data["path"]}',
+            )
+            local_audio_file_path = os.path.join(
+                data_dir, f'{audio_meda_data["task_name"]}_{audio_meda_data["dataset_name"]}_{audio_meda_data["path"]}'
+            )
+            ensure_file_downloaded(source_url=hf_audio_file_path, target_path=local_audio_file_path)
+            if not is_invalid_audio_file(local_audio_file_path):
+                valid_testing_indices.append(row)
+
+        for _, row in enumerate(tqdm(valid_testing_indices)):
+            audio_meda_data_valid = meta_data[row]
+            local_audio_file_path = os.path.join(
+                data_dir,
+                f'{audio_meda_data_valid["task_name"]}'
+                f'_{audio_meda_data_valid["dataset_name"]}_{audio_meda_data_valid["path"]}',
+            )
+
+            answer: str = audio_meda_data_valid["answer_gt"]
+            references: List[Reference] = []
+
+            answer = self._get_label_from_answer(audio_meda_data_valid, answer)
+            # The given correct answer is a letter, but we need an index
+            correct_answer_index: int = ord(answer) - ord("A")
+            # The options are originally appended to the question
+
+            for i, option_key in enumerate(self.OPTION_KEYS):
+                reference: Reference
+                is_correct: bool = i == correct_answer_index
+                reference = Reference(
+                    Output(text=audio_meda_data_valid[option_key]), tags=[CORRECT_TAG] if is_correct else []
+                )
+                references.append(reference)
+
+            input = Input(
+                multimedia_content=MultimediaObject(
+                    [
+                        MediaObject(
+                            content_type=self._get_content_type(audio_meda_data_valid["path"]),
+                            location=local_audio_file_path,
+                        ),
+                        MediaObject(content_type="text/plain", text=audio_meda_data_valid["question"]),
+                    ]
+                )
+            )
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/ami_scenario.py

@@ -0,0 +1,96 @@
+from typing import List
+import os
+import json
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+from helm.common.general import ensure_file_downloaded
+
+
+class AMIScenario(Scenario):
+    """AMI Meeting Corpus
+    The AMI Meeting Corpus (Carletta et al. 2005) is a multi-modal data set consisting of
+    100 hours of meeting recordings. It is being created in the context of a project that
+    is developing meeting browsing technology. The corpus is being recorded using a wide
+    range of devices including close-talking and far-field microphones, individual and
+    room-view video cameras, projection, a whiteboard, and individual pens, all of which
+    produce output signals that are synchronized with each other.
+
+    Paper: https://link.springer.com/chapter/10.1007/11677482_3
+    Code: https://groups.inf.ed.ac.uk/ami/corpus/
+
+    Citation:
+    @inproceedings{Carletta2005TheAM,
+        title={The AMI Meeting Corpus: A Pre-announcement},
+        author={Jean Carletta and Simone Ashby and Sebastien Bourban and Mike Flynn and Ma{\"e}l Guillemot
+        and Thomas Hain and Jaroslav Kadlec and Vasilis Karaiskos and Wessel Kraaij and Melissa Kronenthal
+        and Guillaume Lathoud and Mike Lincoln and Agnes Lisowska Masson and Iain McCowan and Wilfried Post
+        and Dennis Reidsma and Pierre D. Wellner},
+        booktitle={Machine Learning for Multimodal Interaction},
+        year={2005},
+        url={https://api.semanticscholar.org/CorpusID:6118869}
+        }
+    """
+
+    HF_DATASET_NAME = "edinburghcstr/ami"
+    HF_MAPPING_URL = (
+        "https://huggingface.co/datasets/PahaII/SRB_instance_key_mapping/resolve/main/srb_instance_keys.json"
+    )
+    name = "ami"
+    description = (
+        "Speech recognition of speech recorded with different devices "
+        "([Carletta et al, 2005](https://link.springer.com/chapter/10.1007/11677482_3))."
+    )
+    SUBJECT_DICT = {
+        "ihm": {"mapping_key": "ami_ihm_id2line", "srb_mapping": "nearfield"},
+        "sdm": {"mapping_key": "ami_sdm_id2line", "srb_mapping": "farfield"},
+    }
+    tags: List[str] = ["audio", "recognition"]
+
+    def __init__(self, subject: str) -> None:
+        super().__init__()
+        subject = subject.lower()
+        if subject not in AMIScenario.SUBJECT_DICT.keys():
+            raise ValueError(f"Invalid subject. Valid subjects are: {AMIScenario.SUBJECT_DICT.keys()}")
+
+        self._subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        mapping_local_path = os.path.join(output_path, "srb_instance_keys.json")
+        ensure_file_downloaded(source_url=AMIScenario.HF_MAPPING_URL, target_path=mapping_local_path)
+        meta_data = load_dataset(
+            AMIScenario.HF_DATASET_NAME,
+            name=self._subject,
+            cache_dir=output_path,
+            split=TEST_SPLIT,
+        )
+        index_mappings = AMIScenario.SUBJECT_DICT[self._subject]["mapping_key"]
+        srb_mappings = AMIScenario.SUBJECT_DICT[self._subject]["srb_mapping"]
+        mapping_dict = json.load(open(mapping_local_path))
+        srb_mapping_keys = mapping_dict["srb_aim_field_key2audio"][srb_mappings]
+        index2line_num = mapping_dict[index_mappings]
+        for line_num in tqdm(list(srb_mapping_keys)):
+            row = meta_data[int(index2line_num[line_num])]
+            local_audio_name = f"{self._subject}_{line_num}.wav"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+            answer = row["text"].lower()
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/wav", location=local_audio_path)])
+            )
+            references = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/audio_mnist_scenario.py

@@ -0,0 +1,62 @@
+"""Scenarios for audio models"""
+
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+
+
+class AudioMNISTScenario(Scenario):
+    """AudioMNIST
+
+    The AudioMNIST (Becker et al, 2023) dataset consists of a dataset of 30000 audio samples of
+    spoken digits (0-9) of 60 different speakers. The task is to classify the digit from the
+    audio sample.
+
+    Paper: https://arxiv.org/abs/1807.03418
+    Code: https://github.com/soerenab/AudioMNIST
+
+    Citation:
+    @article{audiomnist2023,
+        title = {AudioMNIST: Exploring Explainable Artificial Intelligence for audio analysis on a simple benchmark},
+        journal = {Journal of the Franklin Institute},
+        year = {2023},
+        issn = {0016-0032},
+        doi = {https://doi.org/10.1016/j.jfranklin.2023.11.038},
+        url = {https://www.sciencedirect.com/science/article/pii/S0016003223007536},
+        author = {Sören Becker and Johanna Vielhaben and Marcel Ackermann and Klaus-Robert Müller and Sebastian Lapuschkin and Wojciech Samek},
+        keywords = {Deep learning, Neural networks, Interpretability, Explainable artificial intelligence, Audio classification, Speech recognition},
+    }
+    """  # noqa: E501
+
+    name = "audio_mnist"
+    description = "Classify an audio sample of a spoken digit ([Becker et al, 2023](https://arxiv.org/abs/1807.03418))."
+    tags = ["audio", "classification"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        wav_save_dir: str = os.path.join(output_path, "audio_files")
+        ensure_directory_exists(wav_save_dir)
+        for row in tqdm(load_dataset("flexthink/audiomnist", cache_dir=output_path, split=TEST_SPLIT)):
+            local_audio_path = os.path.join(wav_save_dir, row["audio"]["path"])
+            audio_array = row["audio"]["array"]
+            ensure_audio_file_exists_from_array(local_audio_path, audio_array, row["audio"]["sampling_rate"])
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/wav", location=local_audio_path)])
+            )
+            references = [Reference(Output(text=str(row["digit"])), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/audio_pairs_scenario.py

@@ -0,0 +1,62 @@
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+import json
+
+
+class AudioPAIRSScenario(Scenario):
+    """Audio PAIRS
+
+    Audio PAIRS is an audio extension of the PAIRS dataset (Fraser et al, 2024) to examine gender and
+    racial bias in audio large language models. We convert the questions in the PAIRS dataset to audio
+    clips using OpenAI's TTS-1-HD API.
+
+    This dataset is also modified to add an option to opt-out with "unclear" as a choice.
+    """
+
+    DOWNLOADING_URL = "https://huggingface.co/datasets/UCSC-VLAA/Audio_PAIRS/resolve/main/audio_pairs_files.zip"
+    SUJECTS = ["occupation", "status", "potential_crime"]
+
+    name = "audio_pairs"
+    description = "Examining gender and racial bias in AudioLMs using a converted audio from the PAIRS dataset."
+    tags: List[str] = ["audio", "classification"]
+
+    def __init__(self, subject: str) -> None:
+        super().__init__()
+
+        if subject not in AudioPAIRSScenario.SUJECTS:
+            raise ValueError(f"Invalid subject. Valid subjects are: {AudioPAIRSScenario.SUJECTS}")
+
+        self._subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        downloading_dir: str = os.path.join(output_path, "download")
+        ensure_file_downloaded(source_url=AudioPAIRSScenario.DOWNLOADING_URL, target_path=downloading_dir, unpack=True)
+        data_dir: str = os.path.join(downloading_dir, "audio_pairs_files")
+        audio_file_folder = os.path.join(data_dir, self._subject)
+        audio_instruction_path = os.path.join(data_dir, "audio_pairs_instructions.json")
+        audio_instructions = json.load(open(audio_instruction_path))[self._subject]
+        for audio_file_name, instruction in tqdm(audio_instructions.items()):
+            local_audio_file_name = "_".join(audio_file_name.split("_")[:-1]) + ".mp3"
+            local_audio_path: str = os.path.join(audio_file_folder, local_audio_file_name)
+            content = [
+                MediaObject(content_type="audio/mpeg", location=local_audio_path),
+                MediaObject(content_type="text/plain", text=instruction),
+            ]
+            input = Input(multimedia_content=MultimediaObject(content))
+            references = [Reference(Output(text="unclear"), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/audiocaps_scenario.py

@@ -0,0 +1,59 @@
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+import pandas as pd
+
+
+class AudioCapsScenario(Scenario):
+    """AudioCaps
+
+    AudioCaps is a large-scale dataset of about 46K audio clips to human-written text pairs collected
+    via crowdsourcing on the AudioSet dataset, which covers a wide range of human and animal sounds,
+    musical instruments and genres, and common everyday environmental sounds.
+
+    Paper: https://aclanthology.org/N19-1011.pdf
+    Code: https://github.com/cdjkim/audiocaps
+
+    Citation:
+    @inproceedings{audiocaps,
+        title={AudioCaps: Generating Captions for Audios in The Wild},
+        author={Kim, Chris Dongjoo and Kim, Byeongchang and Lee, Hyunmin and Kim, Gunhee},
+        booktitle={NAACL-HLT},
+        year={2019}
+        }
+    """
+
+    DOWNLOADING_URL = "https://huggingface.co/datasets/Olivia714/audiocaps/resolve/main/wav_files.zip"
+    REFERENCE_URL = "https://huggingface.co/datasets/Olivia714/audiocaps/resolve/main/test.csv"
+
+    name = "audiocaps"
+    description = "A large-scale dataset of about 46K audio clips to human-written text pairs \
+        ([Kim et al, 2019](https://aclanthology.org/N19-1011.pdf))."
+    tags: List[str] = ["audio", "captioning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        data_dir: str = os.path.join(output_path, "wav_files")
+        ensure_file_downloaded(source_url=AudioCapsScenario.DOWNLOADING_URL, target_path=data_dir, unpack=True)
+        for _, row in tqdm(pd.read_csv(AudioCapsScenario.REFERENCE_URL, sep=",").iterrows()):
+            audiocap_id = row["audiocap_id"]
+            audio_path: str = os.path.join(data_dir, f"{audiocap_id}.wav")
+            assert os.path.exists(audio_path), f"Audio file does not exist at path: {audio_path}"
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/wav", location=audio_path)])
+            )
+            references = [Reference(Output(text=str(row["caption"])), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/casual_conversations2_scenario.py

@@ -0,0 +1,152 @@
+from typing import List, Optional
+import os
+
+from tqdm import tqdm
+import json
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+from helm.common.audio_utils import extract_audio
+
+
+class CasualConversations2Scenario(Scenario):
+    """
+    Casual Conversation v2 (Porgali et al, 2023) is composed of over 5,567 participants (26,467 videos).
+    The videos feature paid individuals who agreed to participate in the project and explicitly provided
+    Age, Gender, Language/Dialect, Geo-location, Disability, Physical adornments, Physical attributes labels
+    themselves. The videos were recorded in Brazil, India, Indonesia, Mexico, Philippines, United States,
+    and Vietnam with a diverse set of adults in various categories.
+
+    The dataset contains the audio, speaker's age, gender information in the following languages:
+    English, Hindi, Indonesian, Italian, Portuguese, Spanish, Tagalog, Tamil, Telugu, and Vietnamese.
+
+    Paper: https://arxiv.org/abs/2303.04838
+    Dataset: https://ai.meta.com/datasets/casual-conversations-v2-dataset/
+
+    Requires downloading Causal Conversations V2 from https://ai.meta.com/datasets/casual-conversations-v2-downloads
+
+    Citation:
+    @inproceedings{porgali2023casual,
+    title={The casual conversations v2 dataset},
+    author={Porgali, Bilal and Albiero, V{\'\i}tor and Ryda, Jordan and Ferrer, Cristian Canton and Hazirbas, Caner},
+    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
+    pages={10--17},
+    year={2023}
+    }
+    """
+
+    SUBJECTS = ["age", "gender"]
+    SCRIPT_DOWNLOADING_URL = (
+        "https://huggingface.co/datasets/UCSC-VLAA/Causal_Conversation_V2_script/"
+        "resolve/main/CasualConversationsV2_v2.json"
+    )
+    AGE_INSTRUCTION = "Listen to the audio and take your best guess to estimate the speaker's age."
+    GENDER_INSTRUCTION = "Listen to the audio and take your best guess to determine the speaker's gender."
+    name = "casual_conversations2"
+    description = (
+        "A large scale multilingual speech translation corpus "
+        "([Porgali et al., 2023](https://arxiv.org/abs/2303.04838))."
+    )
+    tags = ["audio", "classification", "multilinguality"]
+    gender_options: List[str] = ["male", "female", "transgender male", "transgender female", "non-binary", "other"]
+    age_options: List[str] = ["18-30", "31-50", "51+", "other"]
+
+    def __init__(self, subject: str) -> None:
+        super().__init__()
+
+        if subject not in self.SUBJECTS:
+            raise ValueError(f"Invalid subject. Valid subjects are: {CasualConversations2Scenario.SUBJECTS}")
+
+        self._subject: str = subject
+        self._convert_answer_to_label_func = (
+            self._convert_age_to_label if subject == "age" else self._convert_gender_to_label
+        )
+        self.options = self.age_options if subject == "age" else self.gender_options
+        self.instruction = self.AGE_INSTRUCTION if subject == "age" else self.GENDER_INSTRUCTION
+
+    def _convert_age_to_label(self, age: str) -> str:
+        if age != "prefer not to say":
+            age_int = int(age)
+            if 18 <= age_int <= 30:
+                return "A"
+            elif 31 <= age_int <= 50:
+                return "B"
+            elif 51 <= age_int:
+                return "C"
+            else:
+                raise ValueError(f"Invalid age: {age}")
+        else:
+            return "D"
+
+    def _convert_gender_to_label(self, gender: Optional[str]) -> str:
+        if gender is not None and gender != "prefer not to say":
+            if gender == "cis man":
+                return "A"
+            elif gender == "cis woman":
+                return "B"
+            elif gender == "transgender man":
+                return "C"
+            elif gender == "transgender woman":
+                return "D"
+            elif gender == "non-binary":
+                return "E"
+            else:
+                raise ValueError(f"Invalid gender: {gender}")
+        else:
+            return "F"
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_dir: str = os.path.join(output_path, "videos_files")
+        assert os.path.exists(data_dir), (
+            f"Download the video files from Meta's Casual Conversations v2 dataset from "
+            f"(https://ai.meta.com/datasets/casual-conversations-v2-downloads) and unzip and place at {data_dir}."
+        )
+        script_file_path: str = os.path.join(output_path, "CasualConversationsV2.json")
+        audio_file_folder: str = os.path.join(output_path, "audio_files")
+        ensure_directory_exists(audio_file_folder)
+        ensure_file_downloaded(self.SCRIPT_DOWNLOADING_URL, script_file_path)
+        audio_scripts = json.load(open(script_file_path))
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        for file_name in tqdm(os.listdir(data_dir)):
+            if file_name.endswith(".mp4"):
+                local_audio_path: str = os.path.join(audio_file_folder, file_name.replace(".mp4", ".mp3"))
+                local_video_path: str = os.path.join(data_dir, file_name)
+
+                if not os.path.exists(local_audio_path):
+                    extract_audio(local_video_path, local_audio_path)
+                assert os.path.exists(local_audio_path), f"Audio file does not exist at path: {local_audio_path}"
+
+                subject_answer = audio_scripts[file_name][self._subject]
+                answer = self._convert_answer_to_label_func(subject_answer)
+                # The given correct answer is a letter, but we need an index
+                correct_answer_index: int = ord(answer) - ord("A")
+                # The options are originally appended to the question
+
+                references: List[Reference] = []
+                for i, option in enumerate(self.options):
+                    reference: Reference
+                    is_correct: bool = i == correct_answer_index
+                    reference = Reference(Output(text=option), tags=[CORRECT_TAG] if is_correct else [])
+                    references.append(reference)
+
+                content = [
+                    MediaObject(content_type="audio/mpeg", location=local_audio_path),
+                    MediaObject(content_type="text/plain", text=self.instruction),
+                ]
+
+                input = Input(multimedia_content=MultimediaObject(content))
+                instances.append(Instance(input=input, references=references, split=split))
+
+        return instances