crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/scenarios/audio_language/corebench_scenario.py

@@ -0,0 +1,77 @@
+from typing import List
+import os
+import json
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+
+
+class COREBenchScenario(Scenario):
+    """COREBench
+
+    COREBench is a new audio benchmark incorporating multi-speaker conversations. It consists of conversational
+    audio, transcript, question, and answer. There are two challenging features of this benchmark: (1) the questions
+    are designed to require reasoning over multiple turns of conversation, and (2) the average audio length is
+    longer than 1 minute, which is significantly longer than existing benchmarks.
+    """
+
+    ANNOT_URL = (
+        "https://huggingface.co/datasets/stanford-crfm/COnversationalREasoningBench_v0.1/resolve/"
+        "main/test/instances.jsonl"
+    )
+    HF_AUDIO_FOLDER = (
+        "https://huggingface.co/datasets/stanford-crfm/COnversationalREasoningBench_v0.1/resolve/main/test/audio"
+    )
+
+    COREBENCH_INSTRUCTION = (
+        "\n\n Answer the question by just giving the final answer and nothing else. "
+        "Answer 'unanswerable' if the question is irrelevant to the audio or cannot be inferred."
+    )
+
+    name = "corebench"
+    description = "Exploring multi-speaker conversational audio reasoning task."
+    tags: List[str] = ["audio", "reasoning"]
+
+    def load_jsonl(self, file_path):
+        with open(file_path, "r", encoding="utf-8") as f:
+            return [json.loads(line.strip()) for line in f]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        annot_save_path = os.path.join(output_path, "instances.jsonl")
+        ensure_file_downloaded(source_url=COREBenchScenario.ANNOT_URL, target_path=annot_save_path)
+        annotations = self.load_jsonl(annot_save_path)
+        audio_save_dir = os.path.join(output_path, "audio")
+        # Download audio files first
+        for row in tqdm(annotations):
+            audio_path = row["audio_path"]
+            local_audio_path = os.path.join(audio_save_dir, audio_path)
+            ensure_file_downloaded(
+                source_url=os.path.join(COREBenchScenario.HF_AUDIO_FOLDER, audio_path), target_path=local_audio_path
+            )
+        for row in tqdm(annotations):
+            local_audio_path = os.path.join(audio_save_dir, row["audio_path"])
+            answer = row["answer"].lower()
+            question = row["question"]
+
+            input = Input(
+                multimedia_content=MultimediaObject(
+                    [
+                        MediaObject(content_type="audio/mpeg", location=local_audio_path),
+                        MediaObject(content_type="text/plain", text=question + self.COREBENCH_INSTRUCTION),
+                    ]
+                )
+            )
+            references = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/mustard_scenario.py

@@ -19,7 +19,7 @@ from helm.common.general import ensure_directory_exists, ensure_file_downloaded
 
 
 class MUStARDScenario(Scenario):
-    """
+    r"""
     MUStARD: Multimodal Sarcasm Detection Dataset
 
     A multimodal video corpus for research in automated sarcasm discovery. The dataset is compiled from popular

helm/benchmark/scenarios/audio_language/ultra_suite_asr_classification_scenario.py

@@ -0,0 +1,104 @@
+from typing import List, Tuple
+import os
+import json
+
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from huggingface_hub import snapshot_download
+
+
+def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
+    """
+    Find all pairs of MP3 and JSON files in the given directory and its subdirectories.
+    Each pair consists of an MP3 file and its corresponding JSON file with the same base name.
+
+    Args:
+        directory: Path to the directory containing the files
+
+    Returns:
+        List of tuples where each tuple contains (mp3_path, json_path)
+    """
+    pairs = []
+
+    # Walk through all directories and subdirectories
+    for root, _, files in os.walk(directory):
+        # Get all MP3 files in current directory
+        mp3_files = [f for f in files if f.endswith(".mp3")]
+
+        for mp3_file in mp3_files:
+            base_name = os.path.splitext(mp3_file)[0]
+            json_file = f"{base_name}.json"
+
+            # Check if corresponding JSON file exists in the same directory
+            if json_file in files:
+                mp3_path = os.path.join(root, mp3_file)
+                json_path = os.path.join(root, json_file)
+                pairs.append((mp3_path, json_path))
+
+    return pairs
+
+
+class UltraSuiteASRClassificationScenario(Scenario):
+    """
+    A scenario for evaluating whether a child speaker has a speech disorder or not.
+    The audio files contain speech from children, potentially with an adult present.
+    The task is to classify whether the child speaker is typically developing or has a speech disorder.
+    """
+
+    name = "speech_disorder"
+    description = "A scenario for evaluating speech disorders in children"
+    tags = ["audio", "classification", "speech_disorder", "asr"]
+
+    # Classification options
+    options: List[str] = ["Healthy", "Unhealthy"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Create instances from the audio files and their corresponding JSON annotations.
+        The data directory should contain:
+        - Audio files (e.g., .mp3)
+        - A JSON file with annotations containing 'answer' field
+        """
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        # Find all pairs of audio and JSON files
+        pairs = find_audio_json_pairs(data_path)
+
+        for audio_path, json_path in tqdm(pairs):
+
+            # Load the annotation
+            with open(json_path, "r") as f:
+                annotation = json.load(f)
+
+            # Get the correct answer and convert to label
+            answer = annotation["disorder_class"]
+            # Create references for each option
+            references: List[Reference] = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
+
+            # Create the input with audio and instruction
+            content = [
+                MediaObject(content_type="audio/mpeg", location=audio_path),
+            ]
+
+            input = Input(multimedia_content=MultimediaObject(content))
+            instances.append(Instance(input=input, references=references, split=split))
+
+        return instances

helm/benchmark/scenarios/audio_language/ultra_suite_asr_transcription_scenario.py

@@ -0,0 +1,99 @@
+from typing import List, Tuple
+import os
+import json
+
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from huggingface_hub import snapshot_download
+
+
+def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
+    """
+    Find all pairs of MP3 and JSON files in the given directory and its subdirectories.
+    Each pair consists of an MP3 file and its corresponding JSON file with the same base name.
+
+    Args:
+        directory: Path to the directory containing the files
+
+    Returns:
+        List of tuples where each tuple contains (mp3_path, json_path)
+    """
+    pairs = []
+
+    # Walk through all directories and subdirectories
+    for root, _, files in os.walk(directory):
+        # Get all MP3 files in current directory
+        mp3_files = [f for f in files if f.endswith(".mp3")]
+
+        for mp3_file in mp3_files:
+            base_name = os.path.splitext(mp3_file)[0]
+            json_file = f"{base_name}.json"
+
+            # Check if corresponding JSON file exists in the same directory
+            if json_file in files:
+                mp3_path = os.path.join(root, mp3_file)
+                json_path = os.path.join(root, json_file)
+                pairs.append((mp3_path, json_path))
+
+    return pairs
+
+
+class UltraSuiteASRTranscriptionScenario(Scenario):
+    """
+    A scenario for evaluating the transcription capabilities of ASR systems.
+    The audio files contain speech from children, potentially with an adult present.
+    The task is to classify whether the child speaker is typically developing or has a speech disorder.
+    """
+
+    name = "speech_disorder"
+    description = "A scenario for evaluating speech disorders in children"
+    tags = ["audio", "transcription", "speech_disorder", "asr"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Create instances from the audio files and their corresponding JSON annotations.
+        The data directory should contain:
+        - Audio files (e.g., .mp3)
+        - A JSON file with annotations containing 'answer' field
+        """
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        # Find all pairs of audio and JSON files
+        pairs = find_audio_json_pairs(data_path)
+
+        for audio_path, json_path in tqdm(pairs):
+
+            # Load the annotation
+            with open(json_path, "r") as f:
+                annotation = json.load(f)
+
+            # Create references for the transcription
+            references: List[Reference] = [Reference(Output(text=annotation["transcription"]), tags=[CORRECT_TAG])]
+
+            # Create the input with audio and instruction
+            content = [
+                MediaObject(content_type="audio/mpeg", location=audio_path),
+            ]
+
+            input = Input(multimedia_content=MultimediaObject(content))
+            instances.append(Instance(input=input, references=references, split=split))
+
+        return instances

helm/benchmark/scenarios/audio_language/ultra_suite_classification_scenario.py

@@ -0,0 +1,118 @@
+from typing import List, Tuple
+import os
+import json
+
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from huggingface_hub import snapshot_download
+
+
+def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
+    """
+    Find all pairs of MP3 and JSON files in the given directory and its subdirectories.
+    Each pair consists of an MP3 file and its corresponding JSON file with the same base name.
+
+    Args:
+        directory: Path to the directory containing the files
+
+    Returns:
+        List of tuples where each tuple contains (mp3_path, json_path)
+    """
+    pairs = []
+
+    # Walk through all directories and subdirectories
+    for root, _, files in os.walk(directory):
+        # Get all MP3 files in current directory
+        mp3_files = [f for f in files if f.endswith(".mp3")]
+
+        for mp3_file in mp3_files:
+            base_name = os.path.splitext(mp3_file)[0]
+            json_file = f"{base_name}.json"
+
+            # Check if corresponding JSON file exists in the same directory
+            if json_file in files:
+                mp3_path = os.path.join(root, mp3_file)
+                json_path = os.path.join(root, json_file)
+                pairs.append((mp3_path, json_path))
+
+    if len(pairs) == 0:
+        raise ValueError(f"No pairs of MP3 and JSON files found in {directory}")
+
+    return pairs
+
+
+class UltraSuiteClassificationScenario(Scenario):
+    """
+    A scenario for evaluating whether a child speaker has a speech disorder or not.
+    The audio files contain speech from children, potentially with an adult present.
+    The task is to classify whether the child speaker is typically developing or has a speech disorder.
+    """
+
+    name = "speech_disorder"
+    description = "A scenario for evaluating speech disorders in children"
+    tags = ["audio", "classification", "speech_disorder"]
+
+    def get_instruction(self, words: str) -> str:
+        return f"""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. The prompt the child is trying to repeat is as follows: {words}. Based on your professional expertise: 1. Assess the child's speech in the recording for signs of typical development or potential speech-language disorder. 2. Conclude your analysis with one of the following labels only: 'typically_developing' or 'speech_disorder'. 3. Provide your response as a single letter without any additional explanation, commentary, or unnecessary text."""  # noqa: E501
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Create instances from the audio files and their corresponding JSON annotations.
+        The data directory should contain:
+        - Audio files (e.g., .mp3)
+        - A JSON file with annotations containing 'answer' field
+        """
+
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        # Find all pairs of audio and JSON files
+        pairs = find_audio_json_pairs(data_path)
+        print(f"Num pairs: {len(pairs)}")
+
+        for audio_path, json_path in tqdm(pairs):
+            # Load the annotation
+            with open(json_path, "r") as f:
+                annotation = json.load(f)
+
+            # Get the correct answer and convert to label
+            answer = annotation["disorder_class"]
+            words = annotation["transcription"]
+            # Create references for each option
+            references: List[Reference] = []
+            correct_label = 0
+            for option in ["typically_developing", "speech_disorder"]:
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == answer else [])
+                references.append(reference)
+                if option == answer:
+                    correct_label += 1
+            if correct_label == 0:
+                continue
+
+            # Create the input with audio and instruction
+            content = [
+                MediaObject(content_type="audio/mpeg", location=audio_path),
+                MediaObject(content_type="text/plain", text=self.get_instruction(words)),
+            ]
+
+            input = Input(multimedia_content=MultimediaObject(content))
+            instances.append(Instance(input=input, references=references, split=split))
+
+        return instances

helm/benchmark/scenarios/audio_language/ultra_suite_disorder_breakdown_scenario.py

@@ -0,0 +1,86 @@
+from typing import List
+import json
+
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from huggingface_hub import snapshot_download
+from .ultra_suite_classification_scenario import find_audio_json_pairs
+
+
+class UltraSuiteDisorderBreakdownScenario(Scenario):
+    """
+    A scenario for evaluating and classifying specific types of speech disorders in children.
+    This scenario extends the basic speech disorder classification by breaking down disorders
+    into specific categories: articulation and phonological disorders.
+    """
+
+    name = "speech_disorder"
+    description = "A scenario for evaluating and classifying specific types of speech disorders in children"
+    tags = ["audio", "classification", "speech_disorder", "disorder_breakdown"]
+
+    def get_instruction(self, words: str) -> str:
+        return f"""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. The prompt text the child is trying to repeat is as follows: {words}. Based on your professional expertise: 1. Assess the child's speech in the recording for signs of typical development or potential speech-language disorder. 2. Conclude your analysis with one of the following labels only: A - 'typically developing' (child's speech patterns and development are within normal age-appropriate ranges), B - 'articulation' (difficulty producing specific speech sounds correctly, such as substituting, omitting, or distorting sounds), C - 'phonological' (difficulty understanding and using the sound system of language, affecting sounds of a particular type). 3. Provide your response as a single letter without any additional explanation, commentary, or unnecessary text."""  # noqa: E501
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Create instances from the audio files and their corresponding JSON annotations.
+        The data directory should contain:
+        - Audio files (e.g., .mp3)
+        - A JSON file with annotations containing 'disorder_class' field
+        """
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        # Find all pairs of audio and JSON files
+        pairs = find_audio_json_pairs(data_path)
+        print(f"Num pairs: {len(pairs)}")
+
+        for audio_path, json_path in tqdm(pairs):
+            # Load the annotation
+            with open(json_path, "r") as f:
+                annotation = json.load(f)
+
+            # Get the correct answer and convert to label
+            if "disorder_type" not in annotation or "transcription" not in annotation:
+                continue
+            label = annotation["disorder_type"]
+            prompt = annotation["transcription"]
+
+            # Create references for each option
+            references: List[Reference] = []
+            correct_label = 0
+            for option in ["typically_developing", "articulation", "phonological"]:
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == label else [])
+                references.append(reference)
+                if option == label:
+                    correct_label += 1
+            if correct_label == 0:
+                continue
+
+            # Create the input with audio and instruction
+            content = [
+                MediaObject(content_type="audio/mpeg", location=audio_path),
+                MediaObject(content_type="text/plain", text=self.get_instruction(prompt)),
+            ]
+
+            input = Input(multimedia_content=MultimediaObject(content))
+            instances.append(Instance(input=input, references=references, split=split))
+
+        return instances

helm/benchmark/scenarios/audio_language/ultra_suite_disorder_symptoms_scenario.py

@@ -0,0 +1,117 @@
+from typing import List, Tuple
+import os
+import json
+
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from huggingface_hub import snapshot_download
+
+
+def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
+    """
+    Find all pairs of MP3 and JSON files in the given directory and its subdirectories.
+    Each pair consists of an MP3 file and its corresponding JSON file with the same base name.
+
+    Args:
+        directory: Path to the directory containing the files
+
+    Returns:
+        List of tuples where each tuple contains (mp3_path, json_path)
+    """
+    pairs = []
+
+    # Walk through all directories and subdirectories
+    for root, _, files in os.walk(directory):
+        # Get all MP3 files in current directory
+        mp3_files = [f for f in files if f.endswith(".mp3")]
+
+        for mp3_file in mp3_files:
+            base_name = os.path.splitext(mp3_file)[0]
+            json_file = f"{base_name}.json"
+
+            # Check if corresponding JSON file exists in the same directory
+            if json_file in files:
+                mp3_path = os.path.join(root, mp3_file)
+                json_path = os.path.join(root, json_file)
+                pairs.append((mp3_path, json_path))
+
+    return pairs
+
+
+class UltraSuiteDisorderSymptomsScenario(Scenario):
+    """
+    A scenario identifying features of speech disorders within the provided audio.
+    The audio files contain speech from children, potentially with an adult present.
+    """
+
+    name = "speech_disorder"
+    description = "A scenario for evaluating speech disorders in children"
+    tags = ["audio", "classification", "speech_disorder"]
+
+    def get_instruction(self, words: str) -> str:
+        prompt = f"""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. The prompt the child is trying to repeat is as follows: {words}. Based on your professional expertise: 1. Assess the child's speech in the recording and recognize any abnormal features in the child's speech. 2. These features can be on of the following: A - 'substitution', B - 'omission', C - 'addition', D - 'typically_developing', or E - 'stuttering'. Here, 'substitution' is when the child substitutes one word/phrase/syllable for another. 'omission' is when the child omits one word/phrase/syllable. 'addition' is when the child adds one word/phrase/syllable. 'typically_developing' is when the child's speech is typical of a child of their age. 'stuttering' is when the child stutters, has difficulty speaking, repeats sounds/words or prolongs sounds/words. 3. Provide your response as a single letter without any additional explanation, commentary, or unnecessary text."""  # noqa: E501
+
+        return prompt
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Create instances from the audio files and their corresponding JSON annotations.
+        The data directory should contain:
+        - Audio files (e.g., .mp3)
+        - A JSON file with annotations containing 'answer' field
+        """
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        # Find all pairs of audio and JSON files
+        pairs = find_audio_json_pairs(data_path)
+
+        for audio_path, json_path in tqdm(pairs):
+
+            # Load the annotation
+            with open(json_path, "r") as f:
+                annotation = json.load(f)
+
+            # Get the correct answer and convert to label
+            if "disorder_symptom" not in annotation or "transcription" not in annotation:
+                continue
+            label = annotation["disorder_symptom"]
+            prompt = annotation["transcription"]
+            # Create references for each option
+            references: List[Reference] = []
+            correct_label = 0
+            for option in ["substitution", "omission", "addition", "typically_developing", "stuttering"]:
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == label else [])
+                references.append(reference)
+                if option == label:
+                    correct_label += 1
+            if correct_label == 0:
+                continue
+
+            # Create the input with audio and instruction
+            content = [
+                MediaObject(content_type="audio/mpeg", location=audio_path),
+                MediaObject(content_type="text/plain", text=self.get_instruction(prompt)),
+            ]
+
+            input = Input(multimedia_content=MultimediaObject(content))
+            instances.append(Instance(input=input, references=references, split=split))
+
+        return instances

helm/benchmark/scenarios/audio_language/vocal_sound_scenario.py

@@ -46,6 +46,10 @@ class VocalSoundScenario(Scenario):
     description = "Classify an audio sample of a spoken digit ([Gong et al, 2022](https://arxiv.org/abs/2205.03433))."
     tags: List[str] = ["audio", "classification"]
 
+    def __init__(self, sound: str) -> None:
+        super().__init__()
+        self._sound: str = sound
+
     def get_instances(self, output_path: str) -> List[Instance]:
         instances: List[Instance] = []
         down_loading_path = os.path.join(output_path, "download")
@@ -53,7 +57,12 @@ class VocalSoundScenario(Scenario):
         wav_save_dir = os.path.join(down_loading_path, "audio_16k")
         for file_name in tqdm(os.listdir(wav_save_dir)):
             local_audio_path: str = os.path.join(wav_save_dir, file_name)
-            if not file_name.endswith(".wav") or is_invalid_audio_file(local_audio_path):
+            if (
+                not file_name.endswith(".wav")
+                or is_invalid_audio_file(local_audio_path)
+                # Skip this problematic file
+                or file_name == "m0083_0_sneeze.wav"
+            ):
                 continue
 
             input = Input(
@@ -61,9 +70,14 @@ class VocalSoundScenario(Scenario):
             )
 
             answer: str = file_name.split("_")[-1].split(".")[0]
+            if answer.lower() != self._sound:
+                continue
+
             if answer == "throatclearing":
                 answer = "throat clearing"
 
             references = [Reference(Output(text=str(answer)), tags=[CORRECT_TAG])]
             instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+
+        assert len(instances) > 0, f"No instances found for sound: {self._sound}"
         return instances

helm/benchmark/scenarios/audio_language/voxceleb2_scenario.py

@@ -40,8 +40,7 @@ class VoxCeleb2Scenario(Scenario):
         "https://huggingface.co/datasets/LAOS-Y/VoxCeleb2-AudioIdentity/resolve/main/voxceleb2_audioidentity.csv"
     )
     IDENTITY_INSTRUCTION = (
-        "Listen to the audio and take your best guess to determine if the two speakers are the same person. "
-        "Give just the letter of your answer and nothing else."
+        "Listen to the audio and take your best guess to determine if the two speakers are the same person."
     )
 
     name = "voxceleb2"