crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/audio_language/multilingual_librispeech_scenario.py

@@ -0,0 +1,80 @@
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+
+
+class MultilingualLibriSpeechScenario(Scenario):
+    """Multilingual Librispeech
+
+    The Multilingual LibriSpeech (Pratap et al, 2020) dataset is derived from read audiobooks
+    from LibriVox and consists of 8 languages, including about 44.5K hours of English and a total
+    of about 6K hours for other 7 languages. The task is to recognize the textual content from the
+    audio sample.
+
+    Paper: https://arxiv.org/abs/2012.03411
+    Code: https://www.openslr.org/
+
+    Citation:
+    @article{Pratap2020MLSAL,
+        title={MLS: A Large-Scale Multilingual Dataset for Speech Research},
+        author={Vineel Pratap and Qiantong Xu and Anuroop Sriram and Gabriel Synnaeve and Ronan Collobert},
+        journal={ArXiv},
+        year={2020},
+        volume={abs/2012.03411}
+        }
+    """
+
+    HF_DATASET_NAME = "facebook/multilingual_librispeech"
+    LANGUAGE_LIST: List[str] = ["dutch", "german", "french", "spanish", "italian", "portuguese", "polish"]
+
+    name = "multilingual_librispeech"
+    description = (
+        "Speech recognition in 7 different languages ([Pratap et al, 2022](https://arxiv.org/abs/2012.03411))."
+    )
+    tags: List[str] = ["audio", "multilinguality", "recognition"]
+
+    def __init__(self, language: str) -> None:
+        super().__init__()
+
+        language = language.lower()
+        if language not in MultilingualLibriSpeechScenario.LANGUAGE_LIST:
+            raise ValueError(f"Invalid language. Valid languages are: {MultilingualLibriSpeechScenario.LANGUAGE_LIST}")
+
+        self._language: str = language
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        for idx, row in enumerate(
+            tqdm(
+                load_dataset(
+                    MultilingualLibriSpeechScenario.HF_DATASET_NAME,
+                    name=self._language,
+                    cache_dir=output_path,
+                    split=TEST_SPLIT,
+                )
+            )
+        ):
+            local_audio_path = os.path.join(audio_save_dir, str(idx) + "_" + row["original_path"].split("/")[-1])
+            # download to the local path
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+            answer = row["transcript"]
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/mpeg", location=local_audio_path)])
+            )
+            references = [Reference(Output(text=str(answer)), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/mustard_scenario.py

@@ -0,0 +1,142 @@
+import json
+import os
+from typing import List
+
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    TEST_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.audio_utils import is_invalid_audio_file, extract_audio
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class MUStARDScenario(Scenario):
+    r"""
+    MUStARD: Multimodal Sarcasm Detection Dataset
+
+    A multimodal video corpus for research in automated sarcasm discovery. The dataset is compiled from popular
+    TV shows including Friends, The Golden Girls, The Big Bang Theory, and Sarcasmaholics Anonymous. MUStARD consists
+    of audiovisual utterances annotated with sarcasm labels. Each utterance is accompanied by its context, providing
+    additional information on the scenario where it occurs.
+
+    We just extract the audio from the given videos.
+
+    The columns of the dataset are:
+    - utterance: The text of the target utterance to classify.
+    - speaker: Speaker of the target utterance.
+    - context: List of utterances (in chronological order) preceding the target utterance.
+    - context_speakers: Respective speakers of the context utterances.
+    - sarcasm: Binary label for sarcasm tag.
+
+    More specifically an example looks like this:
+
+    "1_60": {
+        "utterance": "It's just a privilege to watch your mind at work.",
+        "speaker": "SHELDON",
+        "context": [
+            "I never would have identified the fingerprints of string theory in the aftermath of the Big Bang.",
+            "My apologies. What's your plan?"
+        ],
+        "context_speakers": [
+            "LEONARD",
+            "SHELDON"
+        ],
+        "show": "BBT",
+        "sarcasm": true
+    }
+
+    The key is the video id.
+
+    The video folder has two subfolders:
+    - context_final: Contains the context videos (e.g., 1_60_c.mp4)
+    - utterances_final: Contains the target utterance videos (e.g., 1_60.mp4)
+
+    Citation:
+
+    @inproceedings{mustard,
+        title = "Towards Multimodal Sarcasm Detection (An \_Obviously\_ Perfect Paper)",
+        author = "Castro, Santiago  and
+          Hazarika, Devamanyu  and
+          P{\'e}rez-Rosas, Ver{\'o}nica  and
+          Zimmermann, Roger  and
+          Mihalcea, Rada  and
+          Poria, Soujanya",
+        booktitle = "Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics
+                    (Volume 1: Long Papers)",
+        month = "7",
+        year = "2019",
+        address = "Florence, Italy",
+        publisher = "Association for Computational Linguistics",
+    }
+    """
+
+    RAW_VIDEO_CLIPS_URL: str = "https://huggingface.co/datasets/MichiganNLP/MUStARD/resolve/main/mmsd_raw_data.zip"
+    ANNOTATIONS_URL: str = (
+        "https://raw.githubusercontent.com/soujanyaporia/MUStARD/refs/heads/master/data/" "sarcasm_data.json"
+    )
+
+    name = "mustard"
+    description = "Sarcasm detection benchmark ([Castro et al, 2018](https://arxiv.org/abs/1906.01815))."
+    tags = ["audio", "classification", "toxicity", "sarcasm detection"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the annotations
+        annotations_path: str = os.path.join(output_path, "sarcasm_data.json")
+        ensure_file_downloaded(self.ANNOTATIONS_URL, annotations_path)
+
+        # Where the video files will be downloaded to
+        video_path: str = os.path.join(output_path, "video")
+        ensure_file_downloaded(self.RAW_VIDEO_CLIPS_URL, video_path, unpack=True)
+
+        # Where the audio files will be extracted to
+        audio_path: str = os.path.join(output_path, "audio")
+        ensure_directory_exists(audio_path)
+
+        instances: List[Instance] = []
+        annotations = json.load(open(annotations_path, "r"))
+        for key, row in tqdm(annotations.items()):
+            # Extract the audio from the context video
+            context_audio_path: str = os.path.join(audio_path, f"{key}_c.mp3")
+            if not os.path.exists(context_audio_path):
+                # Extract the audio from the video
+                context_video_path: str = os.path.join(video_path, "context_final", f"{key}_c.mp4")
+                extract_audio(context_video_path, context_audio_path)
+            assert not is_invalid_audio_file(context_audio_path), f"Invalid audio file: {context_audio_path}"
+
+            # Extract the audio from the target utterance video
+            utterance_audio_path: str = os.path.join(audio_path, f"{key}.mp3")
+            if not os.path.exists(utterance_audio_path):
+                utterance_video_path: str = os.path.join(video_path, "utterances_final", f"{key}.mp4")
+                extract_audio(utterance_video_path, utterance_audio_path)
+            assert not is_invalid_audio_file(utterance_audio_path), f"Invalid audio file: {utterance_audio_path}"
+
+            input = Input(
+                multimedia_content=MultimediaObject(
+                    media_objects=[
+                        # Input both the context and the utterance audio
+                        MediaObject(text="Context:", content_type="text/plain"),
+                        MediaObject(location=context_audio_path, content_type="audio/mpeg"),
+                        MediaObject(text="Utterance:", content_type="text/plain"),
+                        MediaObject(location=utterance_audio_path, content_type="audio/mpeg"),
+                        MediaObject(
+                            text="Given the context, does the utterance contain sarcasm?", content_type="text/plain"
+                        ),
+                    ]
+                )
+            )
+            is_sarcastic: bool = row["sarcasm"]
+            references = [
+                Reference(Output(text="Yes"), tags=[CORRECT_TAG] if is_sarcastic else []),
+                Reference(Output(text="No"), tags=[CORRECT_TAG] if not is_sarcastic else []),
+            ]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+
+        return instances

helm/benchmark/scenarios/audio_language/mutox_scenario.py

@@ -0,0 +1,254 @@
+from io import BytesIO
+from typing import List
+import os
+import requests
+
+from pydub import AudioSegment
+from tqdm import tqdm
+import pandas as pd
+
+from helm.benchmark.scenarios.scenario import (
+    TEST_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.audio_utils import is_invalid_audio_file
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+from helm.common.hierarchical_logger import hlog, htrack_block
+
+
+class MuToxScenario(Scenario):
+    """
+    MuTox: MuTox: Universal MUltilingual Audio-based TOXicity Dataset and Zero-shot Detector
+
+    MuTox, the first highly multilingual audio-based dataset with toxicity labels. The dataset consists of 20k
+    audio utterances for English and Spanish, and 4k for the other languages. To showcase the quality of this
+    dataset, we train the MuTox audio-based toxicity classifier, which allows zero-shot toxicity detection across
+    a broad range of languages. This classifier outperforms existing text-based trainable classifiers by more than
+    1% AUC, while increasing the language coverage from 8 to 100+ languages. When compared to a wordlist-based
+    classifier that covers a similar number of languages, MuTox improves precision and recall by ∼2.5 times.
+
+    Languages:
+        "Arabic": "arb",
+        "Bengali": "ben",
+        "Bulgarian": "bul",
+        "Catalan": "cat",
+        "Czech": "ces",
+        "Mandarin Chinese": "cmn",
+        "Danish": "dan",
+        "German": "deu",
+        "Greek": "ell",
+        "English": "eng",
+        "Estonian": "est",
+        "Western Persian": "fas",
+        "Finnish": "fin",
+        "French": "fra",
+        "Hebrew": "heb",
+        "Hindi": "hin",
+        "Hungarian": "hun",
+        "Indonesian": "ind",
+        "Italian": "ita",
+        "Dutch": "nld",
+        "Polish": "pol",
+        "Portuguese": "por",
+        "Russian": "rus",
+        "Spanish": "spa",
+        "Slovak": "slk",
+        "Swahili": "swh",
+        "Tagalog": "tgl",
+        "Turkish": "tur",
+        "Urdu": "urd",
+        "Vietnamese": "vie",
+
+    The columns of the dataset are:
+
+    id: a string id of the segment;
+    lang: 3-letter language code;
+    partition: one of train, dev, or devtest
+    public_url_segment: a string formatted as url:start:end, where start and end are indicated in milliseconds;
+    audio_file_transcript: text transctiption of the segment;
+    contains_toxicity, toxicity_types, perlocutionary_effects: annotation results as strings
+    label: an integer label, equal to 1 if contains_toxicity equals Yes and 0 otherwise;
+    etox_result: toxic word (or multiple words, separated by |) detected by the Etox matcher;
+    detoxify_score: toxicity probabilities predicted by the Detoxify system (float numbers between 0 and 1);
+    mutox_speech_score, mutox_text_score, mutox_zero_shot_speech_score, mutox_zero_shot_text_score: MuTox predictions
+    as float numbers with any value (they can be interpreted as logits,
+    i.e. probabilities before a sigmoid transformation).
+
+    Citation:
+
+    @misc{costajussà2023mutox,
+      title={MuTox: Universal MUltilingual Audio-based TOXicity Dataset and Zero-shot Detector},
+      author={ Marta R. Costa-jussà, Mariano Coria Meglioli, Pierre Andrews, David Dale, Prangthip Hansanti,
+      Elahe Kalbassi, Alex Mourachko, Christophe Ropers, Carleigh Wood},
+      year={2023},
+      eprint={},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+    }
+    """
+
+    ANNOTATIONS_URL = "https://dl.fbaipublicfiles.com/seamless/datasets/mutox.tsv"
+
+    LANGAUGE_CODES = {
+        "Arabic": "arb",
+        "Bengali": "ben",
+        "Bulgarian": "bul",
+        "Catalan": "cat",
+        "Czech": "ces",
+        "Mandarin_Chinese": "cmn",
+        "Danish": "dan",
+        "German": "deu",
+        "Greek": "ell",
+        "English": "eng",
+        "Estonian": "est",
+        "Western_Persian": "fas",
+        "Finnish": "fin",
+        "French": "fra",
+        "Hebrew": "heb",
+        "Hindi": "hin",
+        "Hungarian": "hun",
+        "Indonesian": "ind",
+        "Italian": "ita",
+        "Dutch": "nld",
+        "Polish": "pol",
+        "Portuguese": "por",
+        "Russian": "rus",
+        "Spanish": "spa",
+        "Slovak": "slk",
+        "Swahili": "swh",
+        "Tagalog": "tgl",
+        "Turkish": "tur",
+        "Urdu": "urd",
+        "Vietnamese": "vie",
+    }
+
+    name = "mutox"
+    description = "Toxicity detection benchmark ([Costa-jussà et al, 2024](https://arxiv.org/abs/2401.05060))."
+    tags = ["audio", "classification", "toxicity "]
+
+    @staticmethod
+    def track_bad_audio_file(bad_audio_file: str, output_path: str) -> None:
+        """
+        Many of the links do not exist or point to broken so we keep track of them
+        and skip them in the future runs to significantly speed up gathering the instances.
+        """
+        with open(output_path, "a") as f:
+            f.write(bad_audio_file + "\n")
+
+    def __init__(self, language: str) -> None:
+        super().__init__()
+        self._language_code: str = self.LANGAUGE_CODES[language]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the annotations
+        annotations_path: str = os.path.join(output_path, "mutox.tsv")
+        ensure_file_downloaded(self.ANNOTATIONS_URL, annotations_path)
+
+        # Read bad audio files
+        bad_audio_files: set[str] = set()
+        bad_audio_files_path: str = os.path.join(output_path, "bad_audio_files.txt")
+        if os.path.exists(bad_audio_files_path):
+            # Each line is the audio file name
+            with open(bad_audio_files_path, "r") as f:
+                for line in f:
+                    bad_audio_files.add(line.strip())
+            hlog(f"Found {len(bad_audio_files)} bad audio files.")
+
+        # Where the audio files will be downloaded to
+        audio_path: str = os.path.join(output_path, "audio")
+        ensure_directory_exists(audio_path)
+
+        instances: List[Instance] = []
+        df = pd.read_csv(annotations_path, delimiter="\t")
+        hlog(f"Found {len(df)} rows in the dataset")
+
+        valid_count: int = 0
+        total_count: int = 0
+        for row in tqdm(df.itertuples()):
+            # Only proces examples that are in devtest and the language we're interested in
+            if row.partition != "devtest":
+                continue
+
+            if row.lang != self._language_code:
+                continue
+
+            total_count += 1
+
+            # Discard known bad audio files
+            audio_filename: str = f"{row.id}.mp3"
+            with htrack_block(f"Processing audio file: {audio_filename}"):
+                if audio_filename in bad_audio_files:
+                    hlog(f"Skipping this example -- known bad audio file: {audio_filename}")
+                    continue
+
+                local_audio_path: str = os.path.join(audio_path, audio_filename)
+                if not os.path.exists(local_audio_path):
+                    # The provided URL has the complete audio, so we need to download it and clip it
+                    # public_url_segment: a string formatted as url:start:end,
+                    if not isinstance(row.public_url_segment, str):
+                        # Sometimes URL is just a float causing an error. Skip those.
+                        hlog(f"Skipping this example -- invalid URL: {row.public_url_segment}")
+                        continue
+
+                    parts = row.public_url_segment.split()
+                    if len(parts) != 3:
+                        hlog(f"Skipping this example -- invalid URL: {row.public_url_segment}")
+                        continue
+
+                    audio_url, start_ms_str, end_ms_str = parts
+                    start_ms, end_ms = int(start_ms_str), int(end_ms_str)
+
+                    # Download the full audio file
+                    try:
+                        response = requests.get(audio_url, stream=True)
+                        response.raise_for_status()
+
+                        # Load audio from the downloaded file
+                        audio = AudioSegment.from_file(BytesIO(response.content))
+
+                        # Clip the audio
+                        clipped_audio = audio[start_ms:end_ms]
+
+                        # Save the clipped file
+                        clipped_audio.export(local_audio_path, format="mp3")
+                    except Exception as e:
+                        # Some files are no longer available or invalid
+                        hlog(
+                            f"Skipping this example -- error downloading / processing audio file from {audio_url}: {e}"
+                        )
+                        self.track_bad_audio_file(audio_filename, bad_audio_files_path)
+                        continue
+
+                if is_invalid_audio_file(local_audio_path):
+                    hlog(f"Skipping this example -- Invalid audio file even after downloading at {local_audio_path}")
+                    self.track_bad_audio_file(audio_filename, bad_audio_files_path)
+                    continue
+                else:
+                    valid_count += 1
+                    hlog(f"Valid audio file found at {local_audio_path} (valid/total: {valid_count}/{total_count})")
+
+            input = Input(
+                multimedia_content=MultimediaObject(
+                    media_objects=[
+                        # Following what was prompted annotators as outlined in the paper
+                        MediaObject(text="Does the utterance contain toxicity?", content_type="text/plain"),
+                        MediaObject(location=local_audio_path, content_type="audio/mpeg"),
+                    ]
+                )
+            )
+
+            is_toxic: bool = row.label == 1
+            references = [
+                Reference(Output(text="Yes"), tags=[CORRECT_TAG] if is_toxic else []),
+                Reference(Output(text="No"), tags=[CORRECT_TAG] if not is_toxic else []),
+            ]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+
+        assert len(instances) > 0, f"No instances found for language: {self._language_code}"
+        return instances

helm/benchmark/scenarios/audio_language/parade_scenario.py

@@ -0,0 +1,97 @@
+from typing import List
+import os
+import json
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+
+
+class PARADEScenario(Scenario):
+    """PARADE
+
+    PARADE dataset is inspired by the PAIRS dataset for evaluating occupation and status bias
+    in vision-language models. We collect a new dataset of audio-text multi-choice QA task that
+    involves exploring occupation and status bias. The dataset consists of 436 audio-text QA pairs
+    with 3 options each.
+    """
+
+    ANNOT_URL = (
+        "https://huggingface.co/datasets/UCSC-VLAA/PARADE_audio/resolve/main/audio_result" "_path_mapping_v2.json"
+    )
+    MALE_AUDIO = "https://huggingface.co/datasets/UCSC-VLAA/PARADE_audio/resolve/main/onyx.zip"
+    FEMALE_AUDIO = "https://huggingface.co/datasets/UCSC-VLAA/PARADE_audio/resolve/main/nova.zip"
+
+    PARADE_INSTRUCTION = "\n\n Answer the question with one of the following options: A, B, or C."
+
+    SUBSET_LIST = ["occupation", "status"]
+    VOICE_MAPPING = {"male": "onyx", "female": "nova"}
+
+    name = "parade"
+    description = "Exploring occupation and status bias in the audio-text multi-choice QA task."
+    tags: List[str] = ["audio", "bias"]
+
+    def __init__(self, subset: str, voice: str) -> None:
+        super().__init__()
+
+        subset = subset.lower()
+        voice = voice.lower()
+        if subset not in PARADEScenario.SUBSET_LIST:
+            raise ValueError(f"Invalid subset. Valid subsets are: {PARADEScenario.SUBSET_LIST}")
+
+        if voice not in PARADEScenario.VOICE_MAPPING.keys():
+            raise ValueError(f"Invalid voice. Valid voices are: {PARADEScenario.VOICE_MAPPING.keys()}")
+
+        self._subset: str = subset
+        self._voice: str = voice
+
+    def _convert_answer_to_label(self, options: list, answer: str) -> str:
+        option_list = ["A", "B", "C"]
+        return option_list[options.index(answer)]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        annot_save_path = os.path.join(output_path, "annotation.json")
+        ensure_file_downloaded(source_url=PARADEScenario.ANNOT_URL, target_path=annot_save_path)
+        annotations = json.load(open(annot_save_path))[self.VOICE_MAPPING[self._voice]][self._subset]
+        test_annotations = []
+        for key in annotations:
+            for key2 in annotations[key]:
+                test_annotations.append(annotations[key][key2])
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        if self._voice == "male":
+            ensure_file_downloaded(source_url=PARADEScenario.MALE_AUDIO, target_path=audio_save_dir, unpack=True)
+        else:
+            ensure_file_downloaded(source_url=PARADEScenario.FEMALE_AUDIO, target_path=audio_save_dir, unpack=True)
+        for row in tqdm(test_annotations):
+            local_audio_path = os.path.join(output_path, "audio_files", row["path"])
+            answer = self._convert_answer_to_label(row["options"], row["label"])
+            # The given correct answer is a letter, but we need an index
+            correct_answer_index: int = ord(answer) - ord("A")
+            references: List[Reference] = []
+            question = row["question"]
+            for i, option in enumerate(row["options"]):
+                reference: Reference
+                is_correct: bool = i == correct_answer_index
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if is_correct else [])
+                references.append(reference)
+
+            input = Input(
+                multimedia_content=MultimediaObject(
+                    [
+                        MediaObject(content_type="audio/mpeg", location=local_audio_path),
+                        MediaObject(content_type="text/plain", text=question + self.PARADE_INSTRUCTION),
+                    ]
+                )
+            )
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances