crfm-helm 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/scenarios/audio_language/librispeech_fairness_scenario.py

@@ -0,0 +1,96 @@
+from typing import List
+import os
+import json
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from collections import OrderedDict
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+from helm.common.general import ensure_file_downloaded
+
+
+class LibriSpeechFairnessScenario(Scenario):
+    """LibriSpeech Fairness Scenario
+    The LibriSpeech corpus (Vassil et al. 2015) is derived from audiobooks that are part of the LibriVox
+    project, and contains 1000 hours of speech sampled at 16 kHz. The data has separately prepared language-model
+    training data and pre-built language models. This corpus is one of the most widely-used ASR corpus, which
+    has been extended to many applicaitons such as robust ASR and multilingual ASR tasks.
+
+    Paper: https://ieeexplore.ieee.org/document/7178964
+    Code: https://www.openslr.org/12
+
+    Citation:
+    @INPROCEEDINGS{7178964,
+        author={Panayotov, Vassil and Chen, Guoguo and Povey, Daniel and Khudanpur, Sanjeev},
+        booktitle={2015 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+        title={Librispeech: An ASR corpus based on public domain audio books},
+        year={2015},
+        doi={10.1109/ICASSP.2015.7178964}}
+    """
+
+    HF_DATASET_NAME = "openslr/librispeech_asr"
+    HF_MAPPING_URL = (
+        "https://huggingface.co/datasets/PahaII/SRB_instance_key_mapping/resolve/main/srb_instance_keys.json"
+    )
+    GENDER_MAPPING_URL = (
+        "https://huggingface.co/datasets/PahaII/librispeech_id2gender/resolve/main/librispeech_id2gender.json"
+    )
+    GENDERS = ["male", "female"]
+
+    name = "librispeech_fairness"
+    description = (
+        "Widely-used speech corpus for the speech recognition task "
+        "([Vassil et al. 2015](https://ieeexplore.ieee.org/document/7178964))."
+    )
+    tags: List[str] = ["audio", "recognition"]
+
+    def __init__(self, gender: str) -> None:
+        super().__init__()
+
+        if gender.lower() not in LibriSpeechFairnessScenario.GENDERS:
+            raise ValueError(
+                f"Invalid gender input: {gender}. Valid languages are: {LibriSpeechFairnessScenario.GENDERS}"
+            )
+
+        self._gender: str = gender
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        mapping_local_path = os.path.join(output_path, "librispeech_id2gender.json")
+        ensure_file_downloaded(
+            source_url=LibriSpeechFairnessScenario.GENDER_MAPPING_URL, target_path=mapping_local_path
+        )
+        meta_data = load_dataset(
+            LibriSpeechFairnessScenario.HF_DATASET_NAME,
+            name="clean",
+            cache_dir=output_path,
+            split=TEST_SPLIT,
+        )
+        gender_mapping_dict = json.load(open(mapping_local_path))
+        loading_cases: List[OrderedDict] = []
+        for row in tqdm(meta_data):
+            if gender_mapping_dict[str(row["speaker_id"])] == self._gender.lower():
+                loading_cases.append(row)
+
+        for row in tqdm(loading_cases):
+            local_audio_name = f"librispeech_{row['id']}.mp3"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+            answer = row["text"].lower()
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/mp3", location=local_audio_path)])
+            )
+            references = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/librispeech_scenario.py

@@ -0,0 +1,80 @@
+from typing import List
+import os
+import json
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+from helm.common.general import ensure_file_downloaded
+
+
+class LibriSpeechScenario(Scenario):
+    """LibriSpeech Corpus
+    The LibriSpeech corpus (Vassil et al. 2015) is derived from audiobooks that are part of the LibriVox
+    project, and contains 1000 hours of speech sampled at 16 kHz. The data has separately prepared language-model
+    training data and pre-built language models. This corpus is one of the most widely-used ASR corpus, which
+    has been extended to many applicaitons such as robust ASR and multilingual ASR tasks.
+
+    Paper: https://ieeexplore.ieee.org/document/7178964
+    Code: https://www.openslr.org/12
+
+    Citation:
+    @INPROCEEDINGS{7178964,
+        author={Panayotov, Vassil and Chen, Guoguo and Povey, Daniel and Khudanpur, Sanjeev},
+        booktitle={2015 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
+        title={Librispeech: An ASR corpus based on public domain audio books},
+        year={2015},
+        doi={10.1109/ICASSP.2015.7178964}}
+    """
+
+    HF_DATASET_NAME = "openslr/librispeech_asr"
+    HF_MAPPING_URL = (
+        "https://huggingface.co/datasets/PahaII/SRB_instance_key_mapping/resolve/main/srb_instance_keys.json"
+    )
+    SRB_KEY = "srb_librispeech_noises_key2audio"
+    SRB_SUBSET = "gnoise.1"
+    MAPPING_KEY = "librispeech_id2line"
+
+    name = "librispeech"
+    description = (
+        "Widely-used speech corpus for the speech recognition task "
+        "([Vassil et al. 2015](https://ieeexplore.ieee.org/document/7178964))."
+    )
+    tags: List[str] = ["audio", "recognition"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        mapping_local_path = os.path.join(output_path, "srb_instance_keys.json")
+        ensure_file_downloaded(source_url=LibriSpeechScenario.HF_MAPPING_URL, target_path=mapping_local_path)
+        meta_data = load_dataset(
+            LibriSpeechScenario.HF_DATASET_NAME,
+            name="clean",
+            cache_dir=output_path,
+            split=TEST_SPLIT,
+        )
+        mapping_dict = json.load(open(mapping_local_path))
+        srb_mapping_keys = mapping_dict[self.SRB_KEY][self.SRB_SUBSET]
+        index2line_num = mapping_dict[self.MAPPING_KEY]
+        for line_num in tqdm(list(srb_mapping_keys)):
+            row = meta_data[int(index2line_num[line_num])]
+            local_audio_name = f"{self.name}_{line_num}.mp3"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+            answer = row["text"].lower()
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/mp3", location=local_audio_path)])
+            )
+            references = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/meld_audio_scenario.py

@@ -0,0 +1,113 @@
+import os
+from typing import List
+
+import pandas as pd
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.audio_utils import ensure_audio_file_exists_from_array, get_array_from_audio_file
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class MELDAudioScenario(Scenario):
+    """Multimodal EmotionLines Dataset (MELD) Audio
+
+    Multimodal EmotionLines Dataset (MELD) has been created by enhancing and extending EmotionLines dataset.
+    MELD has more than 1400 dialogues and 13000 utterances from Friends TV series. Multiple speakers participated
+    in the dialogues. Each utterance in a dialogue has been labeled by any of these seven emotions -
+    Anger, Disgust, Sadness, Joy, Neutral, Surprise and Fear.
+
+    The task is to classify the emotion based on only the audio clip.
+
+    Website: https://affective-meld.github.io/
+    Paper: https://arxiv.org/abs/1810.02508
+    Dataset: https://huggingface.co/datasets/DavidCombei/Wav2Vec_MELD_Audio
+
+    Citation:
+    S. Poria, D. Hazarika, N. Majumder, G. Naik, R. Mihalcea,
+    E. Cambria. MELD: A Multimodal Multi-Party Dataset
+    for Emotion Recognition in Conversation. (2018)
+
+    Chen, S.Y., Hsu, C.C., Kuo, C.C. and Ku, L.W.
+    EmotionLines: An Emotion Corpus of Multi-Party
+    Conversations. arXiv preprint arXiv:1802.08379 (2018).
+    """  # noqa: E501
+
+    name = "meld_audio"
+    description = "Classify emotions in audio clips from the television series Friends ([Poria et al, 2018](https://arxiv.org/abs/1810.02508))."  # noqa: E501
+    tags = ["audio", "classification"]
+
+    LABEL_NAMES = ["anger", "disgust", "fear", "joy", "neutral", "sadness", "surprise"]
+    CSV_URL_PREFIX = (
+        "https://raw.githubusercontent.com/declare-lab/MELD/2d2011b409d3ca2d7e94460cd007d434b1d0a102/data/MELD/"
+    )
+    SPLIT_NAME_TO_CSV_FILE_NAME = {
+        VALID_SPLIT: "dev_sent_emo.csv",
+        TRAIN_SPLIT: "train_sent_emo.csv",
+        TEST_SPLIT: "test_sent_emo.csv",
+    }
+    SPLIT_NAME_TO_TGZ_FILE_NAME = {
+        VALID_SPLIT: "audios_validation.tgz",
+        TRAIN_SPLIT: "audios_train.tgz",
+        TEST_SPLIT: "audios_test.tgz",
+    }
+    SAMPLE_RATE = 16000
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        csv_dir = os.path.join(output_path, "csv")
+        ensure_directory_exists(csv_dir)
+
+        instances: List[Instance] = []
+        # Only download the test split.
+        # We don't need the train split (because we use zero shot) or the validation split.
+        split_name = TEST_SPLIT
+
+        # Download the CSV to get the labels and IDs
+        csv_file_name = MELDAudioScenario.SPLIT_NAME_TO_CSV_FILE_NAME[split_name]
+        csv_file_path = os.path.join(csv_dir, csv_file_name)
+        ensure_file_downloaded(MELDAudioScenario.CSV_URL_PREFIX + csv_file_name, csv_file_path)
+        df = pd.read_csv(csv_file_path, header=0).rename(columns={"Sr No.": "serial_number"})
+
+        # Download FLAC files
+        flac_dir = os.path.join(output_path, f"flac_{split_name}")
+        ensure_file_downloaded(
+            source_url=f"https://huggingface.co/datasets/zrr1999/MELD_Text_Audio/resolve/main/archive/{MELDAudioScenario.SPLIT_NAME_TO_TGZ_FILE_NAME[split_name]}?download=true",  # noqa: E501
+            target_path=flac_dir,
+            unpack=True,
+            unpack_type="untar",
+        )
+
+        wav_dir = os.path.join(output_path, f"wav_{split_name}")
+        ensure_directory_exists(wav_dir)
+        for row in tqdm(df.itertuples()):
+            # Transcode FLAC to WAV
+            wav_file_name = f"dia{row.Dialogue_ID}_utt{row.Utterance_ID}.wav"
+            wav_file_path = os.path.join(wav_dir, wav_file_name)
+            if not os.path.isfile(wav_file_path):
+                flac_file_name = f"dia{row.Dialogue_ID}_utt{row.Utterance_ID}.flac"
+                flac_file_path = os.path.join(flac_dir, flac_file_name)
+                audio_array = get_array_from_audio_file(flac_file_path, MELDAudioScenario.SAMPLE_RATE)
+                ensure_audio_file_exists_from_array(wav_file_path, audio_array, MELDAudioScenario.SAMPLE_RATE)
+            input = Input(
+                multimedia_content=MultimediaObject(
+                    media_objects=[MediaObject(location=wav_file_path, content_type="audio/wav")]
+                )
+            )
+            assert row.Emotion in MELDAudioScenario.LABEL_NAMES
+            references = [Reference(output=Output(text=row.Emotion), tags=[CORRECT_TAG])]
+            instance = Instance(
+                id=str(f"awoo{row.serial_number}"), input=input, references=references, split=split_name
+            )
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/audio_language/multilingual_librispeech_scenario.py

@@ -0,0 +1,80 @@
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+
+
+class MultilingualLibriSpeechScenario(Scenario):
+    """Multilingual Librispeech
+
+    The Multilingual LibriSpeech (Pratap et al, 2020) dataset is derived from read audiobooks
+    from LibriVox and consists of 8 languages, including about 44.5K hours of English and a total
+    of about 6K hours for other 7 languages. The task is to recognize the textual content from the
+    audio sample.
+
+    Paper: https://arxiv.org/abs/2012.03411
+    Code: https://www.openslr.org/
+
+    Citation:
+    @article{Pratap2020MLSAL,
+        title={MLS: A Large-Scale Multilingual Dataset for Speech Research},
+        author={Vineel Pratap and Qiantong Xu and Anuroop Sriram and Gabriel Synnaeve and Ronan Collobert},
+        journal={ArXiv},
+        year={2020},
+        volume={abs/2012.03411}
+        }
+    """
+
+    HF_DATASET_NAME = "facebook/multilingual_librispeech"
+    LANGUAGE_LIST: List[str] = ["dutch", "german", "french", "spanish", "italian", "portuguese", "polish"]
+
+    name = "multilingual_librispeech"
+    description = (
+        "Speech recognition in 7 different languages ([Pratap et al, 2022](https://arxiv.org/abs/2012.03411))."
+    )
+    tags: List[str] = ["audio", "multilinguality", "recognition"]
+
+    def __init__(self, language: str) -> None:
+        super().__init__()
+
+        language = language.lower()
+        if language not in MultilingualLibriSpeechScenario.LANGUAGE_LIST:
+            raise ValueError(f"Invalid language. Valid languages are: {MultilingualLibriSpeechScenario.LANGUAGE_LIST}")
+
+        self._language: str = language
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        for idx, row in enumerate(
+            tqdm(
+                load_dataset(
+                    MultilingualLibriSpeechScenario.HF_DATASET_NAME,
+                    name=self._language,
+                    cache_dir=output_path,
+                    split=TEST_SPLIT,
+                )
+            )
+        ):
+            local_audio_path = os.path.join(audio_save_dir, str(idx) + "_" + row["original_path"].split("/")[-1])
+            # download to the local path
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+            answer = row["transcript"]
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/mpeg", location=local_audio_path)])
+            )
+            references = [Reference(Output(text=str(answer)), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/mustard_scenario.py

@@ -0,0 +1,142 @@
+import json
+import os
+from typing import List
+
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    TEST_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.audio_utils import is_invalid_audio_file, extract_audio
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class MUStARDScenario(Scenario):
+    """
+    MUStARD: Multimodal Sarcasm Detection Dataset
+
+    A multimodal video corpus for research in automated sarcasm discovery. The dataset is compiled from popular
+    TV shows including Friends, The Golden Girls, The Big Bang Theory, and Sarcasmaholics Anonymous. MUStARD consists
+    of audiovisual utterances annotated with sarcasm labels. Each utterance is accompanied by its context, providing
+    additional information on the scenario where it occurs.
+
+    We just extract the audio from the given videos.
+
+    The columns of the dataset are:
+    - utterance: The text of the target utterance to classify.
+    - speaker: Speaker of the target utterance.
+    - context: List of utterances (in chronological order) preceding the target utterance.
+    - context_speakers: Respective speakers of the context utterances.
+    - sarcasm: Binary label for sarcasm tag.
+
+    More specifically an example looks like this:
+
+    "1_60": {
+        "utterance": "It's just a privilege to watch your mind at work.",
+        "speaker": "SHELDON",
+        "context": [
+            "I never would have identified the fingerprints of string theory in the aftermath of the Big Bang.",
+            "My apologies. What's your plan?"
+        ],
+        "context_speakers": [
+            "LEONARD",
+            "SHELDON"
+        ],
+        "show": "BBT",
+        "sarcasm": true
+    }
+
+    The key is the video id.
+
+    The video folder has two subfolders:
+    - context_final: Contains the context videos (e.g., 1_60_c.mp4)
+    - utterances_final: Contains the target utterance videos (e.g., 1_60.mp4)
+
+    Citation:
+
+    @inproceedings{mustard,
+        title = "Towards Multimodal Sarcasm Detection (An \_Obviously\_ Perfect Paper)",
+        author = "Castro, Santiago  and
+          Hazarika, Devamanyu  and
+          P{\'e}rez-Rosas, Ver{\'o}nica  and
+          Zimmermann, Roger  and
+          Mihalcea, Rada  and
+          Poria, Soujanya",
+        booktitle = "Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics
+                    (Volume 1: Long Papers)",
+        month = "7",
+        year = "2019",
+        address = "Florence, Italy",
+        publisher = "Association for Computational Linguistics",
+    }
+    """
+
+    RAW_VIDEO_CLIPS_URL: str = "https://huggingface.co/datasets/MichiganNLP/MUStARD/resolve/main/mmsd_raw_data.zip"
+    ANNOTATIONS_URL: str = (
+        "https://raw.githubusercontent.com/soujanyaporia/MUStARD/refs/heads/master/data/" "sarcasm_data.json"
+    )
+
+    name = "mustard"
+    description = "Sarcasm detection benchmark ([Castro et al, 2018](https://arxiv.org/abs/1906.01815))."
+    tags = ["audio", "classification", "toxicity", "sarcasm detection"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the annotations
+        annotations_path: str = os.path.join(output_path, "sarcasm_data.json")
+        ensure_file_downloaded(self.ANNOTATIONS_URL, annotations_path)
+
+        # Where the video files will be downloaded to
+        video_path: str = os.path.join(output_path, "video")
+        ensure_file_downloaded(self.RAW_VIDEO_CLIPS_URL, video_path, unpack=True)
+
+        # Where the audio files will be extracted to
+        audio_path: str = os.path.join(output_path, "audio")
+        ensure_directory_exists(audio_path)
+
+        instances: List[Instance] = []
+        annotations = json.load(open(annotations_path, "r"))
+        for key, row in tqdm(annotations.items()):
+            # Extract the audio from the context video
+            context_audio_path: str = os.path.join(audio_path, f"{key}_c.mp3")
+            if not os.path.exists(context_audio_path):
+                # Extract the audio from the video
+                context_video_path: str = os.path.join(video_path, "context_final", f"{key}_c.mp4")
+                extract_audio(context_video_path, context_audio_path)
+            assert not is_invalid_audio_file(context_audio_path), f"Invalid audio file: {context_audio_path}"
+
+            # Extract the audio from the target utterance video
+            utterance_audio_path: str = os.path.join(audio_path, f"{key}.mp3")
+            if not os.path.exists(utterance_audio_path):
+                utterance_video_path: str = os.path.join(video_path, "utterances_final", f"{key}.mp4")
+                extract_audio(utterance_video_path, utterance_audio_path)
+            assert not is_invalid_audio_file(utterance_audio_path), f"Invalid audio file: {utterance_audio_path}"
+
+            input = Input(
+                multimedia_content=MultimediaObject(
+                    media_objects=[
+                        # Input both the context and the utterance audio
+                        MediaObject(text="Context:", content_type="text/plain"),
+                        MediaObject(location=context_audio_path, content_type="audio/mpeg"),
+                        MediaObject(text="Utterance:", content_type="text/plain"),
+                        MediaObject(location=utterance_audio_path, content_type="audio/mpeg"),
+                        MediaObject(
+                            text="Given the context, does the utterance contain sarcasm?", content_type="text/plain"
+                        ),
+                    ]
+                )
+            )
+            is_sarcastic: bool = row["sarcasm"]
+            references = [
+                Reference(Output(text="Yes"), tags=[CORRECT_TAG] if is_sarcastic else []),
+                Reference(Output(text="No"), tags=[CORRECT_TAG] if not is_sarcastic else []),
+            ]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+
+        return instances