crfm-helm 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/scenarios/aratrust_scenario.py

@@ -0,0 +1,95 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class AraTrustScenario(Scenario):
+    """AraTrust: An Evaluation of Trustworthiness for LLMs in Arabic
+
+    EXPERIMENTAL: This scenario may have future reverse incompatible changes.
+
+    AraTrust is a comprehensive Trustworthiness benchmark for LLMs in Arabic.
+    AraTrust comprises 522 human-written multiple-choice questions addressing
+    diverse dimensions related to truthfulness, ethics, safety, physical health,
+    mental health, unfairness, illegal activities, privacy, and offensive language.
+
+    - https://huggingface.co/datasets/asas-ai/AraTrust
+    - https://arxiv.org/abs/2403.09017
+
+    Citation:
+
+    ```
+    @misc{alghamdi2024aratrustevaluationtrustworthinessllms,
+      title={AraTrust: An Evaluation of Trustworthiness for LLMs in Arabic},
+      author={Emad A. Alghamdi and Reem I. Masoud and Deema Alnuhait and Afnan Y. Alomairi and Ahmed Ashraf and Mohamed Zaytoon},
+      year={2024},
+      eprint={2403.09017},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2403.09017},
+    }
+    ```
+    """  # noqa: E501
+
+    name = "aratrust"
+    description = "aratrust"
+    tags = ["trustworthiness"]
+
+    CATEGORIES = [
+        "Ethics",
+        "Illegal",
+        "Mental Health",
+        "Offensive",
+        "Physical Health",
+        "Privacy",
+        "Trustfulness",
+        "Unfairness",
+    ]
+    OPTION_KEYS = ["A", "B", "C"]
+
+    def __init__(self, category: str):
+        super().__init__()
+        category = category.replace("_", " ")
+        if category not in self.CATEGORIES and category != "all":
+            raise Exception(f"Unknown category {category}")
+        self.category = category
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset: datasets.Dataset = datasets.load_dataset(
+            "asas-ai/AraTrust",
+            revision="d4dd124ed5b90aeb65a7dda7d88e34fb464a31ec",
+            cache_dir=cache_dir,
+            split="test",
+        )
+        instances: List[Instance] = []
+        for row_index, row in enumerate(dataset):
+            if self.category != "all" and self.category != row["Category"]:
+                continue
+            question_text = row["Question"]
+            option_texts = [row[option_key] for option_key in self.OPTION_KEYS if row[option_key]]
+            joined_option_texts = "\n".join(option_texts)
+            input = Input(text=f"{question_text}\n\n{joined_option_texts}\n")
+            references = [Reference(output=Output(text=row["Answer"]), tags=[CORRECT_TAG])]
+            instance = Instance(
+                id=f"id{row_index}",
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/audio_language/casual_conversations2_scenario.py

@@ -19,7 +19,7 @@ from helm.common.audio_utils import extract_audio
 
 
 class CasualConversations2Scenario(Scenario):
-    """
+    r"""
     Casual Conversation v2 (Porgali et al, 2023) is composed of over 5,567 participants (26,467 videos).
     The videos feature paid individuals who agreed to participate in the project and explicitly provided
     Age, Gender, Language/Dialect, Geo-location, Disability, Physical adornments, Physical attributes labels

helm/benchmark/scenarios/audio_language/mustard_scenario.py

@@ -19,7 +19,7 @@ from helm.common.general import ensure_directory_exists, ensure_file_downloaded
 
 
 class MUStARDScenario(Scenario):
-    """
+    r"""
     MUStARD: Multimodal Sarcasm Detection Dataset
 
     A multimodal video corpus for research in automated sarcasm discovery. The dataset is compiled from popular

helm/benchmark/scenarios/audio_language/{ultra_suite_asr_classification.py → ultra_suite_asr_classification_scenario.py}

@@ -14,7 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
 )
 from helm.common.media_object import MediaObject, MultimediaObject
-from helm.common.general import ensure_file_downloaded
+from huggingface_hub import snapshot_download
 
 
 def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
@@ -58,7 +58,6 @@ class UltraSuiteASRClassificationScenario(Scenario):
     name = "speech_disorder"
     description = "A scenario for evaluating speech disorders in children"
     tags = ["audio", "classification", "speech_disorder", "asr"]
-    HF_MAPPING_URL = "https://https://huggingface.co/datasets/SAA-Lab/SLPHelmUltraSuite"
 
     # Classification options
     options: List[str] = ["Healthy", "Unhealthy"]
@@ -70,14 +69,18 @@ class UltraSuiteASRClassificationScenario(Scenario):
         - Audio files (e.g., .mp3)
         - A JSON file with annotations containing 'answer' field
         """
-        print(f"Downloading dataset from {UltraSuiteASRClassificationScenario.HF_MAPPING_URL} to {output_path}")
-        ensure_file_downloaded(source_url=UltraSuiteASRClassificationScenario.HF_MAPPING_URL, target_path=output_path)
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
 
         instances: List[Instance] = []
         split: str = TEST_SPLIT
 
         # Find all pairs of audio and JSON files
-        pairs = find_audio_json_pairs(output_path)
+        pairs = find_audio_json_pairs(data_path)
 
         for audio_path, json_path in tqdm(pairs):
 
@@ -88,9 +91,7 @@ class UltraSuiteASRClassificationScenario(Scenario):
             # Get the correct answer and convert to label
             answer = annotation["disorder_class"]
             # Create references for each option
-            references: List[Reference] = []
-            reference = Reference(Output(text=answer), tags=[CORRECT_TAG])
-            references.append(reference)
+            references: List[Reference] = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
 
             # Create the input with audio and instruction
             content = [

helm/benchmark/scenarios/audio_language/ultra_suite_asr_transcription_scenario.py

@@ -0,0 +1,99 @@
+from typing import List, Tuple
+import os
+import json
+
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from huggingface_hub import snapshot_download
+
+
+def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
+    """
+    Find all pairs of MP3 and JSON files in the given directory and its subdirectories.
+    Each pair consists of an MP3 file and its corresponding JSON file with the same base name.
+
+    Args:
+        directory: Path to the directory containing the files
+
+    Returns:
+        List of tuples where each tuple contains (mp3_path, json_path)
+    """
+    pairs = []
+
+    # Walk through all directories and subdirectories
+    for root, _, files in os.walk(directory):
+        # Get all MP3 files in current directory
+        mp3_files = [f for f in files if f.endswith(".mp3")]
+
+        for mp3_file in mp3_files:
+            base_name = os.path.splitext(mp3_file)[0]
+            json_file = f"{base_name}.json"
+
+            # Check if corresponding JSON file exists in the same directory
+            if json_file in files:
+                mp3_path = os.path.join(root, mp3_file)
+                json_path = os.path.join(root, json_file)
+                pairs.append((mp3_path, json_path))
+
+    return pairs
+
+
+class UltraSuiteASRTranscriptionScenario(Scenario):
+    """
+    A scenario for evaluating the transcription capabilities of ASR systems.
+    The audio files contain speech from children, potentially with an adult present.
+    The task is to classify whether the child speaker is typically developing or has a speech disorder.
+    """
+
+    name = "speech_disorder"
+    description = "A scenario for evaluating speech disorders in children"
+    tags = ["audio", "transcription", "speech_disorder", "asr"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Create instances from the audio files and their corresponding JSON annotations.
+        The data directory should contain:
+        - Audio files (e.g., .mp3)
+        - A JSON file with annotations containing 'answer' field
+        """
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        # Find all pairs of audio and JSON files
+        pairs = find_audio_json_pairs(data_path)
+
+        for audio_path, json_path in tqdm(pairs):
+
+            # Load the annotation
+            with open(json_path, "r") as f:
+                annotation = json.load(f)
+
+            # Create references for the transcription
+            references: List[Reference] = [Reference(Output(text=annotation["transcription"]), tags=[CORRECT_TAG])]
+
+            # Create the input with audio and instruction
+            content = [
+                MediaObject(content_type="audio/mpeg", location=audio_path),
+            ]
+
+            input = Input(multimedia_content=MultimediaObject(content))
+            instances.append(Instance(input=input, references=references, split=split))
+
+        return instances

helm/benchmark/scenarios/audio_language/ultra_suite_classification_scenario.py

@@ -14,7 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
 )
 from helm.common.media_object import MediaObject, MultimediaObject
-from helm.common.general import ensure_file_downloaded
+from huggingface_hub import snapshot_download
 
 
 def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
@@ -61,7 +61,6 @@ class UltraSuiteClassificationScenario(Scenario):
     name = "speech_disorder"
     description = "A scenario for evaluating speech disorders in children"
     tags = ["audio", "classification", "speech_disorder"]
-    HF_MAPPING_URL = "https://https://huggingface.co/datasets/SAA-Lab/SLPHelmUltraSuite"
 
     def get_instruction(self, words: str) -> str:
         return f"""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. The prompt the child is trying to repeat is as follows: {words}. Based on your professional expertise: 1. Assess the child's speech in the recording for signs of typical development or potential speech-language disorder. 2. Conclude your analysis with one of the following labels only: 'typically_developing' or 'speech_disorder'. 3. Provide your response as a single letter without any additional explanation, commentary, or unnecessary text."""  # noqa: E501
@@ -74,14 +73,18 @@ class UltraSuiteClassificationScenario(Scenario):
         - A JSON file with annotations containing 'answer' field
         """
 
-        print(f"Downloading dataset from {UltraSuiteClassificationScenario.HF_MAPPING_URL} to {output_path}")
-        ensure_file_downloaded(source_url=UltraSuiteClassificationScenario.HF_MAPPING_URL, target_path=output_path)
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
 
         instances: List[Instance] = []
         split: str = TEST_SPLIT
 
         # Find all pairs of audio and JSON files
-        pairs = find_audio_json_pairs(output_path)
+        pairs = find_audio_json_pairs(data_path)
         print(f"Num pairs: {len(pairs)}")
 
         for audio_path, json_path in tqdm(pairs):
@@ -94,9 +97,14 @@ class UltraSuiteClassificationScenario(Scenario):
             words = annotation["transcription"]
             # Create references for each option
             references: List[Reference] = []
+            correct_label = 0
             for option in ["typically_developing", "speech_disorder"]:
                 reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == answer else [])
                 references.append(reference)
+                if option == answer:
+                    correct_label += 1
+            if correct_label == 0:
+                continue
 
             # Create the input with audio and instruction
             content = [

helm/benchmark/scenarios/audio_language/ultra_suite_disorder_breakdown_scenario.py

@@ -13,7 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
 )
 from helm.common.media_object import MediaObject, MultimediaObject
-from helm.common.general import ensure_file_downloaded
+from huggingface_hub import snapshot_download
 from .ultra_suite_classification_scenario import find_audio_json_pairs
 
 
@@ -27,7 +27,6 @@ class UltraSuiteDisorderBreakdownScenario(Scenario):
     name = "speech_disorder"
     description = "A scenario for evaluating and classifying specific types of speech disorders in children"
     tags = ["audio", "classification", "speech_disorder", "disorder_breakdown"]
-    HF_MAPPING_URL = "https://https://huggingface.co/datasets/SAA-Lab/SLPHelmManualLabels"
 
     def get_instruction(self, words: str) -> str:
         return f"""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. The prompt text the child is trying to repeat is as follows: {words}. Based on your professional expertise: 1. Assess the child's speech in the recording for signs of typical development or potential speech-language disorder. 2. Conclude your analysis with one of the following labels only: A - 'typically developing' (child's speech patterns and development are within normal age-appropriate ranges), B - 'articulation' (difficulty producing specific speech sounds correctly, such as substituting, omitting, or distorting sounds), C - 'phonological' (difficulty understanding and using the sound system of language, affecting sounds of a particular type). 3. Provide your response as a single letter without any additional explanation, commentary, or unnecessary text."""  # noqa: E501
@@ -39,14 +38,18 @@ class UltraSuiteDisorderBreakdownScenario(Scenario):
         - Audio files (e.g., .mp3)
         - A JSON file with annotations containing 'disorder_class' field
         """
-        print(f"Downloading dataset from {UltraSuiteDisorderBreakdownScenario.HF_MAPPING_URL} to {output_path}")
-        ensure_file_downloaded(source_url=UltraSuiteDisorderBreakdownScenario.HF_MAPPING_URL, target_path=output_path)
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
 
         instances: List[Instance] = []
         split: str = TEST_SPLIT
 
         # Find all pairs of audio and JSON files
-        pairs = find_audio_json_pairs(output_path)
+        pairs = find_audio_json_pairs(data_path)
         print(f"Num pairs: {len(pairs)}")
 
         for audio_path, json_path in tqdm(pairs):
@@ -62,9 +65,14 @@ class UltraSuiteDisorderBreakdownScenario(Scenario):
 
             # Create references for each option
             references: List[Reference] = []
+            correct_label = 0
             for option in ["typically_developing", "articulation", "phonological"]:
                 reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == label else [])
                 references.append(reference)
+                if option == label:
+                    correct_label += 1
+            if correct_label == 0:
+                continue
 
             # Create the input with audio and instruction
             content = [

helm/benchmark/scenarios/audio_language/ultra_suite_disorder_symptoms_scenario.py

@@ -14,7 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
 )
 from helm.common.media_object import MediaObject, MultimediaObject
-from helm.common.general import ensure_file_downloaded
+from huggingface_hub import snapshot_download
 
 
 def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
@@ -57,7 +57,6 @@ class UltraSuiteDisorderSymptomsScenario(Scenario):
     name = "speech_disorder"
     description = "A scenario for evaluating speech disorders in children"
     tags = ["audio", "classification", "speech_disorder"]
-    HF_MAPPING_URL = "https://https://huggingface.co/datasets/SAA-Lab/SLPHelmManualLabels"
 
     def get_instruction(self, words: str) -> str:
         prompt = f"""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. The prompt the child is trying to repeat is as follows: {words}. Based on your professional expertise: 1. Assess the child's speech in the recording and recognize any abnormal features in the child's speech. 2. These features can be on of the following: A - 'substitution', B - 'omission', C - 'addition', D - 'typically_developing', or E - 'stuttering'. Here, 'substitution' is when the child substitutes one word/phrase/syllable for another. 'omission' is when the child omits one word/phrase/syllable. 'addition' is when the child adds one word/phrase/syllable. 'typically_developing' is when the child's speech is typical of a child of their age. 'stuttering' is when the child stutters, has difficulty speaking, repeats sounds/words or prolongs sounds/words. 3. Provide your response as a single letter without any additional explanation, commentary, or unnecessary text."""  # noqa: E501
@@ -71,14 +70,18 @@ class UltraSuiteDisorderSymptomsScenario(Scenario):
         - Audio files (e.g., .mp3)
         - A JSON file with annotations containing 'answer' field
         """
-        print(f"Downloading dataset from {UltraSuiteDisorderSymptomsScenario.HF_MAPPING_URL} to {output_path}")
-        ensure_file_downloaded(source_url=UltraSuiteDisorderSymptomsScenario.HF_MAPPING_URL, target_path=output_path)
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
 
         instances: List[Instance] = []
         split: str = TEST_SPLIT
 
         # Find all pairs of audio and JSON files
-        pairs = find_audio_json_pairs(output_path)
+        pairs = find_audio_json_pairs(data_path)
 
         for audio_path, json_path in tqdm(pairs):
 
@@ -93,9 +96,14 @@ class UltraSuiteDisorderSymptomsScenario(Scenario):
             prompt = annotation["transcription"]
             # Create references for each option
             references: List[Reference] = []
+            correct_label = 0
             for option in ["substitution", "omission", "addition", "typically_developing", "stuttering"]:
                 reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == label else [])
                 references.append(reference)
+                if option == label:
+                    correct_label += 1
+            if correct_label == 0:
+                continue
 
             # Create the input with audio and instruction
             content = [

helm/benchmark/scenarios/babi_qa_scenario.py

@@ -1,6 +1,7 @@
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -139,3 +141,16 @@ class BabiQAScenario(Scenario):
                             story.append(fact)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="babi_qa",
+            display_name="bAbI",
+            description="The bAbI benchmark for measuring understanding and reasoning [(Weston et al., "
+            "2015)](https://arxiv.org/pdf/1502.05698.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="question answering", what="reasoning", when="2015", who="synthetic", language="English"
+            ),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/bbq_scenario.py

@@ -3,6 +3,7 @@ import os
 import random
 from typing import List, Dict, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     DEFAULT_TEST_SIZE,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 AMBIGUOUS_TAG = "ambiguous"
@@ -237,3 +239,16 @@ class BBQScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="bbq",
+            display_name="BBQ (Bias Benchmark for Question Answering)",
+            short_display_name="BBQ",
+            description="The Bias Benchmark for Question Answering (BBQ) for measuring social bias in "
+            "question answering in ambiguous and unambigous context [(Parrish et al., "
+            "2022)](https://aclanthology.org/2022.findings-acl.165/).",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )