crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/audio_language/speech_robust_bench_scenario.py

@@ -0,0 +1,124 @@
+"""Scenarios for audio models"""
+
+from typing import List
+import os
+import json
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+from helm.common.general import ensure_file_downloaded
+
+
+class SpeechRobustBenchScenario(Scenario):
+    """Speech Robust Bench Scenario
+
+    Speech Robust Bench (Shah et al, 2024) is a comprehensive benchmark for evaluating
+    the robustness of ASR models to diverse corruptions. SRB is composed of 114 input
+    perturbations which simulate an heterogeneous range of corruptions that ASR models
+    may encounter when deployed in the wild. In this scenario, we select four subsets
+    in the benchmark for evaluation, each corresponds to a clean version of audio task.
+
+    Paper: https://arxiv.org/abs/2403.07937
+    Code: https://github.com/ahmedshah1494/speech_robust_bench
+
+    Citation:
+    @article{shah2024speech,
+        title={Speech robust bench: A robustness benchmark for speech recognition},
+        author={Shah, Muhammad A and Noguero, David Solans and Heikkila, Mikko A and Raj,
+        Bhiksha and Kourtellis, Nicolas},
+        journal={arXiv preprint arXiv:2403.07937},
+        year={2024}
+        }
+    """
+
+    HF_DATASET_NAME = "mshah1/speech_robust_bench"
+    HF_MAPPING_URL = (
+        "https://huggingface.co/datasets/PahaII/SRB_instance_key_mapping/resolve/main/srb_instance_keys.json"
+    )
+
+    # Select four subsets of the dataset for the benchmark
+    SUBJECTS_DICT = {
+        "ami_far": {
+            "name": "in-the-wild-AMI",
+            "split": "farfield",
+            "type": "audio/wav",
+            "mapping_key": "srb_aim_field_key2audio",
+        },
+        "ami_near": {
+            "name": "in-the-wild-AMI",
+            "split": "nearfield",
+            "type": "audio/wav",
+            "mapping_key": "srb_aim_field_key2audio",
+        },
+        "librispeech_gnoise": {
+            "name": "librispeech_asr-test.clean_pertEval_500_30",
+            "split": "gnoise.1",
+            "type": "audio/mp3",
+            "mapping_key": "srb_librispeech_noises_key2audio",
+        },
+        "librispeech_env_noise": {
+            "name": "librispeech_asr-test.clean_pertEval_500_30",
+            "split": "env_noise_esc50.1",
+            "type": "audio/mp3",
+            "mapping_key": "srb_librispeech_noises_key2audio",
+        },
+    }
+    # There are 30 different perturbation samples for each LibriSpeech ID
+    PERTURBATION_LEVELS = list(range(1, 31))
+    name = "speech_robust_bench"
+    description = (
+        "Speech recognition for 4 datasets with a wide range of corruptions"
+        "([Shah et al, 2024](https://arxiv.org/abs/2403.07937))."
+    )
+    tags: List[str] = ["audio", "recognition", "robustness", "multilinguality"]
+
+    def __init__(self, subject: str, level: int) -> None:
+        super().__init__()
+
+        self._subject = subject
+        if self._subject not in SpeechRobustBenchScenario.SUBJECTS_DICT.keys():
+            raise ValueError(f"Invalid subject. Valid subjects are: {SpeechRobustBenchScenario.SUBJECTS_DICT.keys()}")
+        self._level = level
+        if self._level not in SpeechRobustBenchScenario.PERTURBATION_LEVELS:
+            raise ValueError(f"Invalid level. Valid levels are: {SpeechRobustBenchScenario.PERTURBATION_LEVELS}")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        subject_name = SpeechRobustBenchScenario.SUBJECTS_DICT[self._subject]["name"]
+        subject_split = SpeechRobustBenchScenario.SUBJECTS_DICT[self._subject]["split"]
+        subject_type = SpeechRobustBenchScenario.SUBJECTS_DICT[self._subject]["type"]
+        subject_audio_type = subject_type.split("/")[-1]
+        subject_mapping = SpeechRobustBenchScenario.SUBJECTS_DICT[self._subject]["mapping_key"]
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        mapping_local_path = os.path.join(output_path, "srb_instance_keys.json")
+        ensure_file_downloaded(source_url=SpeechRobustBenchScenario.HF_MAPPING_URL, target_path=mapping_local_path)
+        mapping_keys = json.load(open(mapping_local_path))[subject_mapping][subject_split]
+        meta_data = load_dataset(
+            SpeechRobustBenchScenario.HF_DATASET_NAME,
+            name=subject_name,
+            cache_dir=output_path,
+            split=subject_split,
+        )
+        for line_num in tqdm(list(mapping_keys.keys())):
+            row = meta_data[int(mapping_keys[line_num][self._level - 1])]
+            local_audio_name = f"{self._subject}_{subject_split}_{line_num}.{subject_audio_type}"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+            answer = row["text"].lower()
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type=subject_type, location=local_audio_path)])
+            )
+            references = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/ultra_suite_asr_classification_scenario.py

@@ -0,0 +1,74 @@
+from typing import List
+import os
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+
+
+class UltraSuiteASRClassificationScenario(Scenario):
+    """
+    A scenario for evaluating whether a child speaker has a speech disorder or not.
+    The audio files contain speech from children, potentially with an adult present.
+    The task is to classify whether the child speaker is typically developing or has a speech disorder.
+    """
+
+    name = "speech_disorder"
+    description = "A scenario for evaluating speech disorders in children"
+    tags = ["audio", "classification", "speech_disorder", "asr"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Create instances from the audio files and their corresponding JSON annotations.
+        The data directory should contain:
+        - Audio files (e.g., .mp3)
+        - A JSON file with annotations containing 'answer' field
+        """
+
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        os.makedirs(audio_save_dir, exist_ok=True)
+
+        print("Downloading SAA-Lab/SLPHelmUltraSuitePlus dataset...")
+        dataset = load_dataset("SAA-Lab/SLPHelmUltraSuitePlus")
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        for idx, row in enumerate(tqdm(dataset["train"])):
+
+            label = row["disorder_class"]
+            transcription = row["transcription"]
+
+            unique_id = str(idx)
+            local_audio_name = f"{label}_{unique_id}.mp3"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+
+            # Create references for each option
+            references: List[Reference] = []
+            for option in ["typically_developing", "speech_disorder"]:
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == label else [])
+                references.append(reference)
+
+            # Create the input with audio and instruction
+            content = [
+                MediaObject(content_type="audio/mpeg", location=local_audio_path),
+            ]
+
+            input = Input(multimedia_content=MultimediaObject(content))
+            instances.append(
+                Instance(input=input, references=references, split=split, extra_data={"transcription": transcription})
+            )
+
+        return instances

helm/benchmark/scenarios/audio_language/ultra_suite_asr_transcription_scenario.py

@@ -0,0 +1,70 @@
+from typing import List
+import os
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+
+
+class UltraSuiteASRTranscriptionScenario(Scenario):
+    """
+    A scenario for evaluating the transcription capabilities of ASR systems.
+    The audio files contain speech from children, potentially with an adult present.
+    The task is to classify whether the child speaker is typically developing or has a speech disorder.
+    """
+
+    name = "speech_disorder"
+    description = "A scenario for evaluating speech disorders in children"
+    tags = ["audio", "transcription", "speech_disorder", "asr"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Create instances from the audio files and their corresponding JSON annotations.
+        The data directory should contain:
+        - Audio files (e.g., .mp3)
+        - A JSON file with annotations containing 'answer' field
+        """
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        os.makedirs(audio_save_dir, exist_ok=True)
+
+        print("Downloading SAA-Lab/SLPHelmUltraSuitePlus dataset...")
+        dataset = load_dataset("SAA-Lab/SLPHelmUltraSuitePlus")
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        # Find all pairs of audio and JSON files
+        for idx, row in enumerate(tqdm(dataset["train"])):
+
+            # Load the annotation
+            # Load the annotation
+            label = row["disorder_class"]
+
+            unique_id = str(idx)
+            local_audio_name = f"{label}_{unique_id}.mp3"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+
+            # Create references for each option
+            references: List[Reference] = [Reference(Output(text=row["transcription"]), tags=[CORRECT_TAG])]
+
+            # Create the input with audio and instruction
+            content = [
+                MediaObject(content_type="audio/mpeg", location=local_audio_path),
+            ]
+
+            input = Input(multimedia_content=MultimediaObject(content))
+            instances.append(Instance(input=input, references=references, split=split))
+
+        return instances

helm/benchmark/scenarios/audio_language/ultra_suite_classification_scenario.py

@@ -0,0 +1,79 @@
+from typing import List
+import os
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+
+
+class UltraSuiteClassificationScenario(Scenario):
+    """
+    A scenario for evaluating whether a child speaker has a speech disorder or not.
+    The audio files contain speech from children, potentially with an adult present.
+    The task is to classify whether the child speaker is typically developing or has a speech disorder.
+    """
+
+    name = "speech_disorder"
+    description = "A scenario for evaluating speech disorders in children"
+    tags = ["audio", "classification", "speech_disorder"]
+
+    def get_instruction(self, words: str) -> str:
+        return f"""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. The prompt the child is trying to repeat is as follows: {words}. Based on your professional expertise: 1. Assess the child's speech in the recording for signs of typical development or potential speech-language disorder. 2. Conclude your analysis with one of the following labels only: 'typically_developing' or 'speech_disorder'. 3. Provide your response as a single letter without any additional explanation, commentary, or unnecessary text."""  # noqa: E501
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Create instances from the audio files and their corresponding JSON annotations.
+        The data directory should contain:
+        - Audio files (e.g., .mp3)
+        - A JSON file with annotations containing 'answer' field
+        """
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        os.makedirs(audio_save_dir, exist_ok=True)
+
+        print("Downloading SAA-Lab/SLPHelmUltraSuitePlus dataset...")
+        dataset = load_dataset("SAA-Lab/SLPHelmUltraSuitePlus")
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        for idx, row in enumerate(tqdm(dataset["train"])):
+
+            # Load the annotation
+            label = row["disorder_class"]
+            transcription = row["transcription"]
+
+            unique_id = str(idx)
+            local_audio_name = f"{label}_{unique_id}.mp3"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+
+            # Create references for each option
+            references: List[Reference] = []
+            options = ["typically_developing", "speech_disorder"]
+            if label not in options:
+                continue
+            for option in options:
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == label else [])
+                references.append(reference)
+
+            # Create the input with audio and instruction
+            content = [
+                MediaObject(content_type="audio/mpeg", location=local_audio_path),
+                MediaObject(content_type="text/plain", text=self.get_instruction(transcription)),
+            ]
+
+            input = Input(multimedia_content=MultimediaObject(content))
+            instances.append(Instance(input=input, references=references, split=split))
+
+        return instances

helm/benchmark/scenarios/audio_language/ultra_suite_disorder_breakdown_scenario.py

@@ -0,0 +1,78 @@
+from typing import List
+import os
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+
+
+class UltraSuiteDisorderBreakdownScenario(Scenario):
+    """
+    A scenario for evaluating and classifying specific types of speech disorders in children.
+    This scenario extends the basic speech disorder classification by breaking down disorders
+    into specific categories: articulation and phonological disorders.
+    """
+
+    name = "speech_disorder"
+    description = "A scenario for evaluating and classifying specific types of speech disorders in children"
+    tags = ["audio", "classification", "speech_disorder", "disorder_breakdown"]
+
+    def get_instruction(self, words: str) -> str:
+        return f"""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. The prompt text the child is trying to repeat is as follows: {words}. Based on your professional expertise: 1. Assess the child's speech in the recording for signs of typical development or potential speech-language disorder. 2. Conclude your analysis with one of the following labels only: A - 'typically developing' (child's speech patterns and development are within normal age-appropriate ranges), B - 'articulation' (difficulty producing specific speech sounds correctly, such as substituting, omitting, or distorting sounds), C - 'phonological' (difficulty understanding and using the sound system of language, affecting sounds of a particular type). 3. Provide your response as a single letter without any additional explanation, commentary, or unnecessary text."""  # noqa: E501
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Create instances from the audio files and their corresponding JSON annotations.
+        The data directory should contain:
+        - Audio files (e.g., .mp3)
+        - A JSON file with annotations containing 'disorder_class' field
+        """
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        os.makedirs(audio_save_dir, exist_ok=True)
+
+        print("Downloading SAA-Lab/SLPHelmUltraSuitePlus dataset...")
+        dataset = load_dataset("SAA-Lab/SLPHelmUltraSuitePlus")
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        for idx, row in enumerate(tqdm(dataset["train"])):
+            # Load the annotation
+            label = row["disorder_type"]
+            transcription = row["transcription"]
+
+            unique_id = str(idx)
+            local_audio_name = f"{label}_{unique_id}.mp3"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+
+            # Create references for each option
+            references: List[Reference] = []
+            options = ["typically_developing", "articulation", "phonological"]
+            if label not in options:
+                continue
+            for option in options:
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == label else [])
+                references.append(reference)
+
+            # Create the input with audio and instruction
+            content = [
+                MediaObject(content_type="audio/mpeg", location=local_audio_path),
+                MediaObject(content_type="text/plain", text=self.get_instruction(transcription)),
+            ]
+
+            input = Input(multimedia_content=MultimediaObject(content))
+            instances.append(Instance(input=input, references=references, split=split))
+
+        return instances

helm/benchmark/scenarios/audio_language/ultra_suite_disorder_symptoms_scenario.py

@@ -0,0 +1,78 @@
+from typing import List
+import os
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+
+
+class UltraSuiteDisorderSymptomsScenario(Scenario):
+    """
+    A scenario identifying features of speech disorders within the provided audio.
+    The audio files contain speech from children, potentially with an adult present.
+    """
+
+    name = "speech_disorder"
+    description = "A scenario for evaluating speech disorders in children"
+    tags = ["audio", "classification", "speech_disorder"]
+
+    def get_instruction(self, words: str) -> str:
+        prompt = f"""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. The prompt the child is trying to repeat is as follows: {words}. Based on your professional expertise: 1. Assess the child's speech in the recording and recognize any abnormal features in the child's speech. 2. These features can be on of the following: A - 'substitution', B - 'omission', C - 'addition', D - 'typically_developing', or E - 'stuttering'. Here, 'substitution' is when the child substitutes one word/phrase/syllable for another. 'omission' is when the child omits one word/phrase/syllable. 'addition' is when the child adds one word/phrase/syllable. 'typically_developing' is when the child's speech is typical of a child of their age. 'stuttering' is when the child stutters, has difficulty speaking, repeats sounds/words or prolongs sounds/words. 3. Provide your response as a single letter without any additional explanation, commentary, or unnecessary text."""  # noqa: E501
+
+        return prompt
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Create instances from the audio files and their corresponding JSON annotations.
+        The data directory should contain:
+        - Audio files (e.g., .mp3)
+        - A JSON file with annotations containing 'answer' field
+        """
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        os.makedirs(audio_save_dir, exist_ok=True)
+
+        print("Downloading SAA-Lab/SLPHelmUltraSuitePlus dataset...")
+        dataset = load_dataset("SAA-Lab/SLPHelmUltraSuitePlus")
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        for idx, row in enumerate(tqdm(dataset["train"])):
+            label = row["disorder_symptom"]
+            transcription = row["transcription"]
+
+            unique_id = str(idx)
+            local_audio_name = f"{label}_{unique_id}.mp3"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+
+            # Create references for each option
+            references: List[Reference] = []
+            options = ["substitution", "omission", "addition", "typically_developing", "stuttering"]
+            if label not in options:
+                continue
+            for option in options:
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == label else [])
+                references.append(reference)
+
+            # Create the input with audio and instruction
+            content = [
+                MediaObject(content_type="audio/mpeg", location=local_audio_path),
+                MediaObject(content_type="text/plain", text=self.get_instruction(transcription)),
+            ]
+
+            input = Input(multimedia_content=MultimediaObject(content))
+            instances.append(Instance(input=input, references=references, split=split))
+
+        return instances

helm/benchmark/scenarios/audio_language/vocal_sound_scenario.py

@@ -0,0 +1,83 @@
+"""Scenarios for audio models"""
+
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+from helm.common.audio_utils import is_invalid_audio_file
+
+
+class VocalSoundScenario(Scenario):
+    """Vocal Sound Scenario
+
+    The VocalSound (Gong et al, 2022) dataset consists of 21,000 crowdsourced recordings
+    of laughter, sighs, coughs, throat clearing, sneezes, and sniffs from 3,365 unique subjects.
+    The task is to classify the human behaviour from the audio sample.
+
+    Paper: https://arxiv.org/abs/2205.03433
+    Code: https://github.com/YuanGongND/vocalsound
+
+    Citation:
+    @INPROCEEDINGS{gong_vocalsound,
+                author={Gong, Yuan and Yu, Jin and Glass, James},
+                booktitle={ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+                title={Vocalsound: A Dataset for Improving Human Vocal Sounds Recognition},
+                year={2022},
+                pages={151-155},
+                doi={10.1109/ICASSP43922.2022.9746828}
+                }
+    """  # noqa: E501
+
+    DOWNLOADING_URL = "https://www.dropbox.com/s/c5ace70qh1vbyzb/vs_release_16k.zip"
+
+    name = "vocal_sound"
+    description = "Classify an audio sample of a spoken digit ([Gong et al, 2022](https://arxiv.org/abs/2205.03433))."
+    tags: List[str] = ["audio", "classification"]
+
+    def __init__(self, sound: str) -> None:
+        super().__init__()
+        self._sound: str = sound
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        down_loading_path = os.path.join(output_path, "download")
+        ensure_file_downloaded(VocalSoundScenario.DOWNLOADING_URL, down_loading_path, unpack=True)
+        wav_save_dir = os.path.join(down_loading_path, "audio_16k")
+        for file_name in tqdm(os.listdir(wav_save_dir)):
+            local_audio_path: str = os.path.join(wav_save_dir, file_name)
+            if (
+                not file_name.endswith(".wav")
+                or is_invalid_audio_file(local_audio_path)
+                # Skip this problematic file
+                or file_name == "m0083_0_sneeze.wav"
+            ):
+                continue
+
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/wav", location=local_audio_path)])
+            )
+
+            answer: str = file_name.split("_")[-1].split(".")[0]
+            if answer.lower() != self._sound:
+                continue
+
+            if answer == "throatclearing":
+                answer = "throat clearing"
+
+            references = [Reference(Output(text=str(answer)), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+
+        assert len(instances) > 0, f"No instances found for sound: {self._sound}"
+        return instances

helm/benchmark/scenarios/audio_language/voice_jailbreak_attacks_scenario.py

@@ -0,0 +1,87 @@
+import os
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    TEST_SPLIT,
+    Scenario,
+    Instance,
+    Input,
+)
+from helm.common.audio_utils import is_invalid_audio_file
+from helm.common.media_object import MediaObject, MultimediaObject
+
+
+class VoiceJailbreakAttacksScenario(Scenario):
+    """
+    Voice Jailbreak Attacks Against GPT-4o
+
+    Paper: https://arxiv.org/abs/2405.19103
+
+    The examples used in this scenario was generated following the instructions in the GitHub:
+    https://github.com/TrustAIRLab/VoiceJailbreakAttack
+    Note: The more advanced jailbreaking subset addresses the AI as "ChatGPT".
+
+    We ran:
+    python tts/prompt2audio.py --dataset baseline --voice fable
+    python tts/prompt2audio.py --dataset baseline --voice fable
+
+    then placed the generated folders at benchmark_output/scenarios/voice_jailbreak_attacks:
+
+    voice_jailbreak_attacks/
+        baseline_fable/
+            *.wav
+        textjailbreak_fable/
+            *.wav
+
+    Base prompts (30 total):
+    https://github.com/TrustAIRLab/VoiceJailbreakAttack/blob/main/data/question_set/questions_tiny.csv
+    Jailbreak templates:
+    https://github.com/TrustAIRLab/VoiceJailbreakAttack/blob/main/data/jailbreak_prompts/text_jailbreak_prompts.csv
+
+    The authors used Attack Success Rate (ASR) as the evaluation metric.
+    Specifically, two authors manually label each response to determine if it answers the forbidden
+    question. If there is any uncertainty, the authors discuss to reach a conclusion
+
+    Citation:
+
+    @article{SWBZ24,
+      author = {Xinyue Shen and Yixin Wu and Michael Backes and Yang Zhang},
+      title = {{Voice Jailbreak Attacks Against GPT-4o}},
+      journal = {{CoRR abs/2405.19103}},
+      year = {2024}
+    }
+    """
+
+    VALID_SUBSETS = ["baseline", "textjailbreak"]
+
+    name = "voice_jailbreak_attacks"
+    description = "Voice jailbreak attacks against GPT-4o ([Shen et al, 2024](https://arxiv.org/abs/2405.19103))."
+    tags = ["audio", "safety"]
+
+    def __init__(self, subset: str = "baseline") -> None:
+        super().__init__()
+        assert subset in self.VALID_SUBSETS, f"Invalid subset: {subset}"
+        self._subset = subset
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        audio_directory_path: str = os.path.join(output_path, f"{self._subset}_fable")
+        assert os.path.exists(audio_directory_path), f"Audio directory does not exist: {audio_directory_path}"
+
+        instances: List[Instance] = []
+        for file in os.listdir(audio_directory_path):
+            if not file.endswith("wav"):
+                continue
+
+            audio_path: str = os.path.join(audio_directory_path, file)
+            assert not is_invalid_audio_file(audio_path), f"Invalid audio file: {audio_path}"
+
+            input = Input(
+                multimedia_content=MultimediaObject(
+                    media_objects=[
+                        MediaObject(location=audio_path, content_type="audio/wav"),
+                    ]
+                )
+            )
+            instances.append(Instance(input=input, references=[], split=TEST_SPLIT))
+
+        return instances