crfm-helm 0.5.4__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/scenarios/audio_language/ami_scenario.py

@@ -0,0 +1,96 @@
+from typing import List
+import os
+import json
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+from helm.common.general import ensure_file_downloaded
+
+
+class AMIScenario(Scenario):
+    """AMI Meeting Corpus
+    The AMI Meeting Corpus (Carletta et al. 2005) is a multi-modal data set consisting of
+    100 hours of meeting recordings. It is being created in the context of a project that
+    is developing meeting browsing technology. The corpus is being recorded using a wide
+    range of devices including close-talking and far-field microphones, individual and
+    room-view video cameras, projection, a whiteboard, and individual pens, all of which
+    produce output signals that are synchronized with each other.
+
+    Paper: https://link.springer.com/chapter/10.1007/11677482_3
+    Code: https://groups.inf.ed.ac.uk/ami/corpus/
+
+    Citation:
+    @inproceedings{Carletta2005TheAM,
+        title={The AMI Meeting Corpus: A Pre-announcement},
+        author={Jean Carletta and Simone Ashby and Sebastien Bourban and Mike Flynn and Ma{\"e}l Guillemot
+        and Thomas Hain and Jaroslav Kadlec and Vasilis Karaiskos and Wessel Kraaij and Melissa Kronenthal
+        and Guillaume Lathoud and Mike Lincoln and Agnes Lisowska Masson and Iain McCowan and Wilfried Post
+        and Dennis Reidsma and Pierre D. Wellner},
+        booktitle={Machine Learning for Multimodal Interaction},
+        year={2005},
+        url={https://api.semanticscholar.org/CorpusID:6118869}
+        }
+    """
+
+    HF_DATASET_NAME = "edinburghcstr/ami"
+    HF_MAPPING_URL = (
+        "https://huggingface.co/datasets/PahaII/SRB_instance_key_mapping/resolve/main/srb_instance_keys.json"
+    )
+    name = "ami"
+    description = (
+        "Speech recognition of speech recorded with different devices "
+        "([Carletta et al, 2005](https://link.springer.com/chapter/10.1007/11677482_3))."
+    )
+    SUBJECT_DICT = {
+        "ihm": {"mapping_key": "ami_ihm_id2line", "srb_mapping": "nearfield"},
+        "sdm": {"mapping_key": "ami_sdm_id2line", "srb_mapping": "farfield"},
+    }
+    tags: List[str] = ["audio", "recognition"]
+
+    def __init__(self, subject: str) -> None:
+        super().__init__()
+        subject = subject.lower()
+        if subject not in AMIScenario.SUBJECT_DICT.keys():
+            raise ValueError(f"Invalid subject. Valid subjects are: {AMIScenario.SUBJECT_DICT.keys()}")
+
+        self._subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        mapping_local_path = os.path.join(output_path, "srb_instance_keys.json")
+        ensure_file_downloaded(source_url=AMIScenario.HF_MAPPING_URL, target_path=mapping_local_path)
+        meta_data = load_dataset(
+            AMIScenario.HF_DATASET_NAME,
+            name=self._subject,
+            cache_dir=output_path,
+            split=TEST_SPLIT,
+        )
+        index_mappings = AMIScenario.SUBJECT_DICT[self._subject]["mapping_key"]
+        srb_mappings = AMIScenario.SUBJECT_DICT[self._subject]["srb_mapping"]
+        mapping_dict = json.load(open(mapping_local_path))
+        srb_mapping_keys = mapping_dict["srb_aim_field_key2audio"][srb_mappings]
+        index2line_num = mapping_dict[index_mappings]
+        for line_num in tqdm(list(srb_mapping_keys)):
+            row = meta_data[int(index2line_num[line_num])]
+            local_audio_name = f"{self._subject}_{line_num}.wav"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
+            answer = row["text"].lower()
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/wav", location=local_audio_path)])
+            )
+            references = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/audio_mnist_scenario.py

@@ -0,0 +1,62 @@
+"""Scenarios for audio models"""
+
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
+
+
+class AudioMNISTScenario(Scenario):
+    """AudioMNIST
+
+    The AudioMNIST (Becker et al, 2023) dataset consists of a dataset of 30000 audio samples of
+    spoken digits (0-9) of 60 different speakers. The task is to classify the digit from the
+    audio sample.
+
+    Paper: https://arxiv.org/abs/1807.03418
+    Code: https://github.com/soerenab/AudioMNIST
+
+    Citation:
+    @article{audiomnist2023,
+        title = {AudioMNIST: Exploring Explainable Artificial Intelligence for audio analysis on a simple benchmark},
+        journal = {Journal of the Franklin Institute},
+        year = {2023},
+        issn = {0016-0032},
+        doi = {https://doi.org/10.1016/j.jfranklin.2023.11.038},
+        url = {https://www.sciencedirect.com/science/article/pii/S0016003223007536},
+        author = {Sören Becker and Johanna Vielhaben and Marcel Ackermann and Klaus-Robert Müller and Sebastian Lapuschkin and Wojciech Samek},
+        keywords = {Deep learning, Neural networks, Interpretability, Explainable artificial intelligence, Audio classification, Speech recognition},
+    }
+    """  # noqa: E501
+
+    name = "audio_mnist"
+    description = "Classify an audio sample of a spoken digit ([Becker et al, 2023](https://arxiv.org/abs/1807.03418))."
+    tags = ["audio", "classification"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        wav_save_dir: str = os.path.join(output_path, "audio_files")
+        ensure_directory_exists(wav_save_dir)
+        for row in tqdm(load_dataset("flexthink/audiomnist", cache_dir=output_path, split=TEST_SPLIT)):
+            local_audio_path = os.path.join(wav_save_dir, row["audio"]["path"])
+            audio_array = row["audio"]["array"]
+            ensure_audio_file_exists_from_array(local_audio_path, audio_array, row["audio"]["sampling_rate"])
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/wav", location=local_audio_path)])
+            )
+            references = [Reference(Output(text=str(row["digit"])), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/audio_pairs_scenario.py

@@ -0,0 +1,62 @@
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+import json
+
+
+class AudioPAIRSScenario(Scenario):
+    """Audio PAIRS
+
+    Audio PAIRS is an audio extension of the PAIRS dataset (Fraser et al, 2024) to examine gender and
+    racial bias in audio large language models. We convert the questions in the PAIRS dataset to audio
+    clips using OpenAI's TTS-1-HD API.
+
+    This dataset is also modified to add an option to opt-out with "unclear" as a choice.
+    """
+
+    DOWNLOADING_URL = "https://huggingface.co/datasets/UCSC-VLAA/Audio_PAIRS/resolve/main/audio_pairs_files.zip"
+    SUJECTS = ["occupation", "status", "potential_crime"]
+
+    name = "audio_pairs"
+    description = "Examining gender and racial bias in AudioLMs using a converted audio from the PAIRS dataset."
+    tags: List[str] = ["audio", "classification"]
+
+    def __init__(self, subject: str) -> None:
+        super().__init__()
+
+        if subject not in AudioPAIRSScenario.SUJECTS:
+            raise ValueError(f"Invalid subject. Valid subjects are: {AudioPAIRSScenario.SUJECTS}")
+
+        self._subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        downloading_dir: str = os.path.join(output_path, "download")
+        ensure_file_downloaded(source_url=AudioPAIRSScenario.DOWNLOADING_URL, target_path=downloading_dir, unpack=True)
+        data_dir: str = os.path.join(downloading_dir, "audio_pairs_files")
+        audio_file_folder = os.path.join(data_dir, self._subject)
+        audio_instruction_path = os.path.join(data_dir, "audio_pairs_instructions.json")
+        audio_instructions = json.load(open(audio_instruction_path))[self._subject]
+        for audio_file_name, instruction in tqdm(audio_instructions.items()):
+            local_audio_file_name = "_".join(audio_file_name.split("_")[:-1]) + ".mp3"
+            local_audio_path: str = os.path.join(audio_file_folder, local_audio_file_name)
+            content = [
+                MediaObject(content_type="audio/mpeg", location=local_audio_path),
+                MediaObject(content_type="text/plain", text=instruction),
+            ]
+            input = Input(multimedia_content=MultimediaObject(content))
+            references = [Reference(Output(text="unclear"), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/audiocaps_scenario.py

@@ -0,0 +1,59 @@
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+import pandas as pd
+
+
+class AudioCapsScenario(Scenario):
+    """AudioCaps
+
+    AudioCaps is a large-scale dataset of about 46K audio clips to human-written text pairs collected
+    via crowdsourcing on the AudioSet dataset, which covers a wide range of human and animal sounds,
+    musical instruments and genres, and common everyday environmental sounds.
+
+    Paper: https://aclanthology.org/N19-1011.pdf
+    Code: https://github.com/cdjkim/audiocaps
+
+    Citation:
+    @inproceedings{audiocaps,
+        title={AudioCaps: Generating Captions for Audios in The Wild},
+        author={Kim, Chris Dongjoo and Kim, Byeongchang and Lee, Hyunmin and Kim, Gunhee},
+        booktitle={NAACL-HLT},
+        year={2019}
+        }
+    """
+
+    DOWNLOADING_URL = "https://huggingface.co/datasets/Olivia714/audiocaps/resolve/main/wav_files.zip"
+    REFERENCE_URL = "https://huggingface.co/datasets/Olivia714/audiocaps/resolve/main/test.csv"
+
+    name = "audiocaps"
+    description = "A large-scale dataset of about 46K audio clips to human-written text pairs \
+        ([Kim et al, 2019](https://aclanthology.org/N19-1011.pdf))."
+    tags: List[str] = ["audio", "captioning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        data_dir: str = os.path.join(output_path, "wav_files")
+        ensure_file_downloaded(source_url=AudioCapsScenario.DOWNLOADING_URL, target_path=data_dir, unpack=True)
+        for _, row in tqdm(pd.read_csv(AudioCapsScenario.REFERENCE_URL, sep=",").iterrows()):
+            audiocap_id = row["audiocap_id"]
+            audio_path: str = os.path.join(data_dir, f"{audiocap_id}.wav")
+            assert os.path.exists(audio_path), f"Audio file does not exist at path: {audio_path}"
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/wav", location=audio_path)])
+            )
+            references = [Reference(Output(text=str(row["caption"])), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances

helm/benchmark/scenarios/audio_language/casual_conversations2_scenario.py

@@ -0,0 +1,152 @@
+from typing import List, Optional
+import os
+
+from tqdm import tqdm
+import json
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+from helm.common.audio_utils import extract_audio
+
+
+class CasualConversations2Scenario(Scenario):
+    """
+    Casual Conversation v2 (Porgali et al, 2023) is composed of over 5,567 participants (26,467 videos).
+    The videos feature paid individuals who agreed to participate in the project and explicitly provided
+    Age, Gender, Language/Dialect, Geo-location, Disability, Physical adornments, Physical attributes labels
+    themselves. The videos were recorded in Brazil, India, Indonesia, Mexico, Philippines, United States,
+    and Vietnam with a diverse set of adults in various categories.
+
+    The dataset contains the audio, speaker's age, gender information in the following languages:
+    English, Hindi, Indonesian, Italian, Portuguese, Spanish, Tagalog, Tamil, Telugu, and Vietnamese.
+
+    Paper: https://arxiv.org/abs/2303.04838
+    Dataset: https://ai.meta.com/datasets/casual-conversations-v2-dataset/
+
+    Requires downloading Causal Conversations V2 from https://ai.meta.com/datasets/casual-conversations-v2-downloads
+
+    Citation:
+    @inproceedings{porgali2023casual,
+    title={The casual conversations v2 dataset},
+    author={Porgali, Bilal and Albiero, V{\'\i}tor and Ryda, Jordan and Ferrer, Cristian Canton and Hazirbas, Caner},
+    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
+    pages={10--17},
+    year={2023}
+    }
+    """
+
+    SUBJECTS = ["age", "gender"]
+    SCRIPT_DOWNLOADING_URL = (
+        "https://huggingface.co/datasets/UCSC-VLAA/Causal_Conversation_V2_script/"
+        "resolve/main/CasualConversationsV2_v2.json"
+    )
+    AGE_INSTRUCTION = "Listen to the audio and take your best guess to estimate the speaker's age."
+    GENDER_INSTRUCTION = "Listen to the audio and take your best guess to determine the speaker's gender."
+    name = "casual_conversations2"
+    description = (
+        "A large scale multilingual speech translation corpus "
+        "([Porgali et al., 2023](https://arxiv.org/abs/2303.04838))."
+    )
+    tags = ["audio", "classification", "multilinguality"]
+    gender_options: List[str] = ["male", "female", "transgender male", "transgender female", "non-binary", "other"]
+    age_options: List[str] = ["18-30", "31-50", "51+", "other"]
+
+    def __init__(self, subject: str) -> None:
+        super().__init__()
+
+        if subject not in self.SUBJECTS:
+            raise ValueError(f"Invalid subject. Valid subjects are: {CasualConversations2Scenario.SUBJECTS}")
+
+        self._subject: str = subject
+        self._convert_answer_to_label_func = (
+            self._convert_age_to_label if subject == "age" else self._convert_gender_to_label
+        )
+        self.options = self.age_options if subject == "age" else self.gender_options
+        self.instruction = self.AGE_INSTRUCTION if subject == "age" else self.GENDER_INSTRUCTION
+
+    def _convert_age_to_label(self, age: str) -> str:
+        if age != "prefer not to say":
+            age_int = int(age)
+            if 18 <= age_int <= 30:
+                return "A"
+            elif 31 <= age_int <= 50:
+                return "B"
+            elif 51 <= age_int:
+                return "C"
+            else:
+                raise ValueError(f"Invalid age: {age}")
+        else:
+            return "D"
+
+    def _convert_gender_to_label(self, gender: Optional[str]) -> str:
+        if gender is not None and gender != "prefer not to say":
+            if gender == "cis man":
+                return "A"
+            elif gender == "cis woman":
+                return "B"
+            elif gender == "transgender man":
+                return "C"
+            elif gender == "transgender woman":
+                return "D"
+            elif gender == "non-binary":
+                return "E"
+            else:
+                raise ValueError(f"Invalid gender: {gender}")
+        else:
+            return "F"
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_dir: str = os.path.join(output_path, "videos_files")
+        assert os.path.exists(data_dir), (
+            f"Download the video files from Meta's Casual Conversations v2 dataset from "
+            f"(https://ai.meta.com/datasets/casual-conversations-v2-downloads) and unzip and place at {data_dir}."
+        )
+        script_file_path: str = os.path.join(output_path, "CasualConversationsV2.json")
+        audio_file_folder: str = os.path.join(output_path, "audio_files")
+        ensure_directory_exists(audio_file_folder)
+        ensure_file_downloaded(self.SCRIPT_DOWNLOADING_URL, script_file_path)
+        audio_scripts = json.load(open(script_file_path))
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        for file_name in tqdm(os.listdir(data_dir)):
+            if file_name.endswith(".mp4"):
+                local_audio_path: str = os.path.join(audio_file_folder, file_name.replace(".mp4", ".mp3"))
+                local_video_path: str = os.path.join(data_dir, file_name)
+
+                if not os.path.exists(local_audio_path):
+                    extract_audio(local_video_path, local_audio_path)
+                assert os.path.exists(local_audio_path), f"Audio file does not exist at path: {local_audio_path}"
+
+                subject_answer = audio_scripts[file_name][self._subject]
+                answer = self._convert_answer_to_label_func(subject_answer)
+                # The given correct answer is a letter, but we need an index
+                correct_answer_index: int = ord(answer) - ord("A")
+                # The options are originally appended to the question
+
+                references: List[Reference] = []
+                for i, option in enumerate(self.options):
+                    reference: Reference
+                    is_correct: bool = i == correct_answer_index
+                    reference = Reference(Output(text=option), tags=[CORRECT_TAG] if is_correct else [])
+                    references.append(reference)
+
+                content = [
+                    MediaObject(content_type="audio/mpeg", location=local_audio_path),
+                    MediaObject(content_type="text/plain", text=self.instruction),
+                ]
+
+                input = Input(multimedia_content=MultimediaObject(content))
+                instances.append(Instance(input=input, references=references, split=split))
+
+        return instances

helm/benchmark/scenarios/audio_language/common_voice_15_scenario.py

@@ -0,0 +1,99 @@
+"""Scenarios for audio models"""
+
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from collections import OrderedDict
+from tqdm import tqdm
+from datasets import load_dataset
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.hierarchical_logger import hlog
+
+
+class CommonVoice15Scenario(Scenario):
+    """CommonVoice15 Scenario
+
+    The most recent release of CommonVoice15 (Ardila et al, 2019) includes 114 languages. Over 50,000
+    individuals have participated so far, resulting in 2,500 hours of collected audio. This is the largest
+    audio corpus in the public domain for speech recognition, both in terms of number of hours and number
+    of languages. The task is to recognize the speech from the audio sample.
+
+
+
+    Paper: https://arxiv.org/abs/1912.06670
+    Code: https://github.com/common-voice/common-voice
+
+    Citation:
+    @article{ardila2019common,
+        title={Common voice: A massively-multilingual speech corpus},
+        author={Ardila, Rosana and Branson, Megan and Davis, Kelly and
+        Henretty, Michael and Kohler, Michael and Meyer, Josh and Morais,
+        Reuben and Saunders, Lindsay and Tyers, Francis M and Weber, Gregor},
+        journal={arXiv preprint arXiv:1912.06670},
+        year={2019}
+        }
+
+    """
+
+    HF_DATASET_NAME = "mozilla-foundation/common_voice_15_0"
+
+    # Randomly selected 4 languages from 114 languages in the Common Voice 15 dataset following
+    # Qwen2-Audio (https://arxiv.org/abs/2407.10759). The full language is:
+    # https://huggingface.co/datasets/mozilla-foundation/common_voice_15_0/blob/main/languages.py
+    _COMMON_VOICE_TEST_LANG_TO_ID = OrderedDict(
+        [
+            ("English", "en"),
+            ("Chinese_hk", "zh-HK"),
+            ("German", "de"),
+            ("French", "fr"),
+        ]
+    )
+
+    name = "common_voice_15"
+    description = "Speech recognition for 4 languages from 114 different languages in Common Voice 15 \
+        ([Ardila et al, 2019](https://arxiv.org/abs/1912.06670))."
+    tags: List[str] = ["audio", "recognition", "multilinguality"]
+
+    def __init__(self, language: str) -> None:
+        super().__init__()
+
+        language = language.capitalize()
+        if language not in CommonVoice15Scenario._COMMON_VOICE_TEST_LANG_TO_ID.keys():
+            raise ValueError(
+                f"Invalid language. Valid languages are: {CommonVoice15Scenario._COMMON_VOICE_TEST_LANG_TO_ID.keys()}"
+            )
+
+        self._language: str = language
+        hlog(
+            "You need to sign in Huggingface to download the dataset. Please remember "
+            "to sign in to download the dataset."
+        )
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        language_category = CommonVoice15Scenario._COMMON_VOICE_TEST_LANG_TO_ID[self._language]
+        for row in tqdm(
+            load_dataset(
+                CommonVoice15Scenario.HF_DATASET_NAME,
+                name=language_category,
+                cache_dir=output_path,
+                split=TEST_SPLIT,
+                trust_remote_code=True,
+            )
+        ):
+            local_audio_path = row["path"]
+            answer = row["sentence"]
+            input = Input(
+                multimedia_content=MultimediaObject([MediaObject(content_type="audio/mpeg", location=local_audio_path)])
+            )
+            references = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+        return instances