crfm-helm 0.5.6__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/run_specs/lmkt_run_specs.py

@@ -0,0 +1,144 @@
+"""Run spec functions for Vietnam WVS cultural alignment evaluation."""
+
+from helm.benchmark.adaptation.common_adapter_specs import (
+    get_generation_adapter_spec,
+)
+from helm.benchmark.metrics.common_metric_specs import (
+    get_exact_match_metric_specs,
+    get_f1_metric_specs,
+    get_open_ended_generation_metric_specs,
+)
+from helm.benchmark.metrics.lmkt_metric_specs import get_semantic_similarity_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+INSTRUCTIONS = {
+    "cultural_value_understanding_wvs": {
+        "en": {
+            "instructions": "Please respond as the {country} persona described below.",
+            "input_noun": "Question",
+            "output_noun": "Answer",
+        },
+        "vi": {
+            "instructions": "Vui lòng trả lời như một người {country} được mô tả bên dưới.",
+            "input_noun": "Câu hỏi",
+            "output_noun": "Trả lời",
+        },
+    },
+    "social_norm_application_normad": {
+        "en": {
+            "instructions": "Please respond as the {country} persona described below.",
+            "input_noun": "Situation",
+            "output_noun": "Response",
+        },
+        "vi": {
+            "instructions": "Vui lòng trả lời như một người {country} được mô tả bên dưới.",
+            "input_noun": "Tình huống",
+            "output_noun": "Phản hồi",
+        },
+    },
+    "social_norm_explanation_normad": {
+        "en": {
+            "instructions": "Please respond as the {country} persona described below.",
+            "input_noun": "Situation",
+            "output_noun": "Explanation",
+        },
+        "vi": {
+            "instructions": "Vui lòng trả lời như một người {country} được mô tả bên dưới.",
+            "input_noun": "Tình huống",
+            "output_noun": "Giải thích",
+        },
+    },
+}
+
+COUNTRIES = {
+    "US": "United States",
+    "VN": "Vietnam",
+}
+
+
+@run_spec_function("cultural_value_understanding_wvs")
+def get_cultural_value_understanding_wvs_spec(language: str, country: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.lmkt_scenarios.CulturalValueUnderstandingWVSScenario",
+        args={
+            "language": language,
+            "num_personas": 300,
+            "num_question_variants": 4,
+            "include_few_shot_examples": True,
+        },
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=INSTRUCTIONS["cultural_value_understanding_wvs"][language]["instructions"].format(
+            country=COUNTRIES[country]
+        ),
+        input_noun=INSTRUCTIONS["cultural_value_understanding_wvs"][language]["input_noun"],
+        output_noun=INSTRUCTIONS["cultural_value_understanding_wvs"][language]["output_noun"],
+        max_tokens=3,
+        stop_sequences=[],
+    )
+
+    return RunSpec(
+        name="cultural_value_understanding_wvs",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs() + get_f1_metric_specs(),
+        groups=["lmkt", "cultural_value_understanding_wvs"],
+    )
+
+
+@run_spec_function("social_norm_application_normad")
+def get_social_norm_application_normad_spec(language: str, country: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.lmkt_scenarios.SocialNormApplicationNormADScenario",
+        args={
+            "language": language,
+        },
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=INSTRUCTIONS["social_norm_application_normad"][language]["instructions"].format(
+            country=COUNTRIES[country]
+        ),
+        input_noun=INSTRUCTIONS["social_norm_application_normad"][language]["input_noun"],
+        output_noun=INSTRUCTIONS["social_norm_application_normad"][language]["output_noun"],
+        max_tokens=5,
+        stop_sequences=[],
+    )
+
+    return RunSpec(
+        name="social_norm_application_normad",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs() + get_f1_metric_specs(),
+        groups=["lmkt", "social_norm_application_normad"],
+    )
+
+
+@run_spec_function("social_norm_explanation_normad")
+def get_social_norm_explanation_normad_spec(language: str, country: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.lmkt_scenarios.SocialNormExplanationNormADScenario",
+        args={
+            "language": language,
+        },
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=INSTRUCTIONS["social_norm_explanation_normad"][language]["instructions"].format(
+            country=COUNTRIES[country]
+        ),
+        input_noun=INSTRUCTIONS["social_norm_explanation_normad"][language]["input_noun"],
+        output_noun=INSTRUCTIONS["social_norm_explanation_normad"][language]["output_noun"],
+        max_tokens=128,
+        stop_sequences=[],
+    )
+
+    return RunSpec(
+        name="social_norm_explanation_normad",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_open_ended_generation_metric_specs() + get_semantic_similarity_metric_specs(),
+        groups=["lmkt", "social_norm_explanation_normad"],
+    )

helm/benchmark/run_specs/long_context_run_specs.py

@@ -1,4 +1,9 @@
-from helm.benchmark.adaptation.adapter_spec import ADAPT_CHAT, ADAPT_GENERATION, AdapterSpec
+from helm.benchmark.adaptation.adapter_spec import (
+    ADAPT_CHAT,
+    ADAPT_GENERATION,
+    ADAPT_MULTIPLE_CHOICE_JOINT,
+    AdapterSpec,
+)
 from helm.benchmark.metrics.common_metric_specs import (
     get_exact_match_metric_specs,
     get_open_ended_generation_metric_specs,
@@ -29,6 +34,27 @@ def _get_long_context_generation_adapter_spec(max_tokens: int) -> AdapterSpec:
     )
 
 
+def _get_long_context_multiple_choice_adapter_spec(max_tokens: int) -> AdapterSpec:
+    return AdapterSpec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        global_prefix="",
+        global_suffix="",
+        instructions="Read the passage and answer the following question. Respond with only a single letter corresponding to your choice.",  # noqa: E501
+        input_prefix="",
+        input_suffix="\n",
+        reference_prefix="A. ",
+        reference_suffix="\n",
+        output_prefix="",
+        output_suffix="",
+        instance_prefix="",
+        max_train_instances=0,
+        num_outputs=1,
+        temperature=0.0,
+        max_tokens=max_tokens,
+        stop_sequences=[],
+    )
+
+
 @run_spec_function("ruler_hotpotqa")
 def get_ruler_hotpotqa_spec(max_num_words: int = 131072) -> RunSpec:
     scenario_spec = ScenarioSpec(
@@ -96,6 +122,27 @@ def get_infinite_bench_en_qa_spec(max_num_words: int = 131072) -> RunSpec:
     )
 
 
+@run_spec_function("infinite_bench_en_mc")
+def get_infinite_bench_en_mc_spec(max_num_words: int = 131072) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.infinite_bench_en_mc_scenario.InfiniteBenchEnMCScenario",
+        args={
+            "max_num_words": max_num_words,
+        },
+    )
+
+    adapter_spec = _get_long_context_multiple_choice_adapter_spec(max_tokens=40)
+    metric_specs = get_exact_match_metric_specs()
+
+    return RunSpec(
+        name=f"infinite_bench_en_mc:max_num_words={max_num_words}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=["infinite_bench_en_mc"],
+    )
+
+
 @run_spec_function("infinite_bench_en_sum")
 def get_infinite_bench_en_sum_spec(max_num_words: int = 131072) -> RunSpec:
 

helm/benchmark/run_specs/multilingual_run_specs.py

@@ -0,0 +1,50 @@
+from helm.benchmark.adaptation.adapter_spec import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("mmmlu")
+def get_mmmlu_spec(locale: str, subject: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.mmmlu_scenario.MMMLUScenario", args={"locale": locale, "subject": subject}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions=f"The following are multiple choice questions (with answers) about {subject.replace('_', ' ')}. Answer the last question. Respond only with only a single letter corresponding to your choice.",  # noqa: E501
+        input_noun="Question",
+        output_noun="Answer",
+    )
+
+    return RunSpec(
+        name=f"mmmlu:locale={locale},subject={subject}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["mmmlu", f"mmmlu_{locale}_{subject}"],
+    )
+
+
+@run_spec_function("exams_multilingual")
+def get_exams_multilingual_spec(language: str, subject: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.exams_multilingual_scenario.EXAMSMultilingualScenario",
+        args={"language": language, "subject": subject},
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions=f"The following are multiple choice questions (with answers) about {subject.replace('_', ' ')}. Answer the last question. Respond only with only a single letter corresponding to your choice.",  # noqa: E501
+        input_noun="Question",
+        output_noun="Answer",
+    )
+
+    return RunSpec(
+        name=f"exams_multilingual:locale={language},subject={subject}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["exams_multilingual", f"exams_multilingual_{language}_{subject}"],
+    )

helm/benchmark/run_specs/speech_disorder_audio_run_specs.py

@@ -73,9 +73,7 @@ def get_ultra_suite_classification_run_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.audio_language.ultra_suite_classification_scenario.UltraSuiteClassificationScenario",  # noqa: E501
     )
-    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
-        input_noun=None, output_noun="Answer", max_train_instances=0
-    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(input_noun=None, output_noun="Answer")
     metric_specs: List[MetricSpec] = audio_classification_metric_specs()
     run_spec_name: str = "ultra_suite_classification"
     return RunSpec(
@@ -92,9 +90,7 @@ def get_ultra_suite_disorder_breakdown_run_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.audio_language.ultra_suite_disorder_breakdown_scenario.UltraSuiteDisorderBreakdownScenario",  # noqa: E501
     )
-    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
-        input_noun=None, output_noun="Answer", max_train_instances=0
-    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(input_noun=None, output_noun="Answer")
     metric_specs: List[MetricSpec] = audio_classification_metric_specs()
     run_spec_name: str = "ultra_suite_classification_breakdown"
     return RunSpec(
@@ -112,7 +108,7 @@ def get_ultra_suite_disorder_breakdown_run_spec() -> RunSpec:
 @run_spec_function("ultra_suite_asr_classification")
 def get_ultra_suite_asr_classification_run_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_asr_classification.UltraSuiteASRClassificationScenario",  # noqa: E501
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_asr_classification_scenario.UltraSuiteASRClassificationScenario",  # noqa: E501
     )
     adapter_spec = _get_generation_adapter_spec(
         instructions="""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording is provided to you, typically consisting of a speech prompt from a pathologist followed by a child's repetition. Based on your expertise transcribe the child's speech into text. Do not make any assumptions about the words the child is expected to say. Only transcribe based on the words that the child actually says. Only respond with the text transcription, no other text or commentary.""",  # noqa: E501
@@ -133,7 +129,7 @@ def get_ultra_suite_asr_classification_run_spec() -> RunSpec:
 @run_spec_function("ultra_suite_asr_transcription")
 def get_ultra_suite_asr_transcription_run_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_asr_classification.UltraSuiteASRClassificationScenario",  # noqa: E501
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_asr_transcription_scenario.UltraSuiteASRTranscriptionScenario",  # noqa: E501
     )
     adapter_spec = _get_generation_adapter_spec(
         instructions="""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. Based on your expertise transcribe the child's speech into text. Try to understand what the child is expected to say. And only respond with the transcription of the child's speech. Not the pathologist's prompt or any other commentary. Only respond with the text transcription, no other text, commentary or punctuations.""",  # noqa: E501
@@ -155,9 +151,7 @@ def get_ultra_suite_disorder_symptoms_run_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.audio_language.ultra_suite_disorder_symptoms_scenario.UltraSuiteDisorderSymptomsScenario",  # noqa: E501
     )
-    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
-        input_noun=None, output_noun="Answer", max_train_instances=0
-    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(input_noun=None, output_noun="Answer")
     metric_specs: List[MetricSpec] = audio_classification_metric_specs()
     run_spec_name: str = "ultra_suite_disorder_symptoms"
     return RunSpec(

helm/benchmark/scenarios/alghafa_scenario.py

@@ -0,0 +1,126 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class AlGhafaScenario(Scenario):
+    """AlGhafa Evaluation Benchmark for Arabic Language Models
+
+    EXPERIMENTAL: This scenario may have future reverse incompatible changes.
+
+    Multiple-choice evaluation benchmark for zero- and few-shot evaluation of Arabic LLMs,
+    consisting of
+
+    - https://huggingface.co/datasets/OALL/AlGhafa-Arabic-LLM-Benchmark-Native/
+    - https://aclanthology.org/2023.arabicnlp-1.21/
+
+    Citation:
+
+    ```
+    @inproceedings{almazrouei-etal-2023-alghafa,
+        title = "{A}l{G}hafa Evaluation Benchmark for {A}rabic Language Models",
+        author = "Almazrouei, Ebtesam  and
+        Cojocaru, Ruxandra  and
+        Baldo, Michele  and
+        Malartic, Quentin  and
+        Alobeidli, Hamza  and
+        Mazzotta, Daniele  and
+        Penedo, Guilherme  and
+        Campesan, Giulia  and
+        Farooq, Mugariya  and
+        Alhammadi, Maitha  and
+        Launay, Julien  and
+        Noune, Badreddine",
+        editor = "Sawaf, Hassan  and
+        El-Beltagy, Samhaa  and
+        Zaghouani, Wajdi  and
+        Magdy, Walid  and
+        Abdelali, Ahmed  and
+        Tomeh, Nadi  and
+        Abu Farha, Ibrahim  and
+        Habash, Nizar  and
+        Khalifa, Salam  and
+        Keleg, Amr  and
+        Haddad, Hatem  and
+        Zitouni, Imed  and
+        Mrini, Khalil  and
+        Almatham, Rawan",
+        booktitle = "Proceedings of ArabicNLP 2023",
+        month = dec,
+        year = "2023",
+        address = "Singapore (Hybrid)",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2023.arabicnlp-1.21/",
+        doi = "10.18653/v1/2023.arabicnlp-1.21",
+        pages = "244--275",
+        abstract = "Recent advances in the space of Arabic large language models have opened up a wealth of potential practical applications. From optimal training strategies, large scale data acquisition and continuously increasing NLP resources, the Arabic LLM landscape has improved in a very short span of time, despite being plagued by training data scarcity and limited evaluation resources compared to English. In line with contributing towards this ever-growing field, we introduce AlGhafa, a new multiple-choice evaluation benchmark for Arabic LLMs. For showcasing purposes, we train a new suite of models, including a 14 billion parameter model, the largest monolingual Arabic decoder-only model to date. We use a collection of publicly available datasets, as well as a newly introduced HandMade dataset consisting of 8 billion tokens. Finally, we explore the quantitative and qualitative toxicity of several Arabic models, comparing our models to existing public Arabic LLMs."
+    }
+    ```
+    """  # noqa: E501
+
+    name = "alghafa"
+    description = "AlGhafa"
+    tags = ["multiple choice"]
+
+    HF_SPLIT_TO_HELM_SPLIT = {"validation": TRAIN_SPLIT, "test": TEST_SPLIT}
+    REFERENCE_PREFIX = "sol"
+
+    def __init__(self, subset: str):
+        super().__init__()
+        self.subset = subset
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
+            self.subset,
+            revision="a31ebd34ca311d7e0cfc6ad7f458b3435af280f5",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            option_indexes = [
+                int(s.removeprefix(self.REFERENCE_PREFIX))
+                for s in dataset[0].keys()
+                if s.startswith(self.REFERENCE_PREFIX)
+            ]
+            for row_index, row in enumerate(dataset):
+                input = Input(text=row["query"])
+                references: List[Reference] = []
+                # Need to add 1 because label is zero-indexed and has a value from 0 to (N - 1),
+                # but column names are 1 indexed and have values from "sol1" to "solN"
+                correct_option_index = int(row["label"]) + 1
+                for option_index in option_indexes:
+                    column_name = f"{self.REFERENCE_PREFIX}{option_index}"
+                    references.append(
+                        Reference(
+                            output=Output(text=row[column_name]),
+                            tags=[CORRECT_TAG] if option_index == correct_option_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=f"id{row_index}_{split_name}",
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/arabic_mmlu_scenario.py

@@ -0,0 +1,78 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class ArabicMMLUScenario(Scenario):
+    """ArabicMMLU
+
+    EXPERIMENTAL: This scenario may have future reverse incompatible changes.
+
+    ArabicMMLU is the first multi-task language understanding benchmark
+    for Arabic language, sourced from school exams across diverse educational
+    levels in different countries spanning North Africa, the Levant, and the
+    Gulf regions. The data comprises 40 tasks and 14,575 multiple-choice questions
+    in Modern Standard Arabic (MSA), and is carefully constructed by collaborating
+    with native speakers in the region.
+
+    - https://huggingface.co/datasets/MBZUAI/ArabicMMLU
+    - https://aclanthology.org/2024.findings-acl.334/
+    """
+
+    name = "arabic_mmlu"
+    description = "Arabic Massive Multitask Language Understanding"
+    tags = ["knowledge", "multiple_choice"]
+
+    OPTIONS = ["A", "B", "C", "D"]
+    HF_SPLIT_TO_HELM_SPLIT = {"dev": TRAIN_SPLIT, "test": TEST_SPLIT}
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "MBZUAI/ArabicMMLU",
+            "All",
+            revision="7aa530e2893ac420352b3f5c1a1310c010e9758b",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            for row_index, row in enumerate(dataset):
+                input = Input(text=row["Question"])
+                references: List[Reference] = []
+                correct_option_index = ord(row["Answer Key"]) - ord("A") + 1
+                for option_index in range(1, 6):
+                    column_name = f"Option {option_index}"
+                    if not row[column_name]:
+                        continue
+                    references.append(
+                        Reference(
+                            output=Output(text=row[column_name]),
+                            tags=[CORRECT_TAG] if option_index == correct_option_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=f"id{row_index}",
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/aratrust_scenario.py

@@ -0,0 +1,76 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class AraTrustScenario(Scenario):
+    """AraTrust: An Evaluation of Trustworthiness for LLMs in Arabic
+
+    EXPERIMENTAL: This scenario may have future reverse incompatible changes.
+
+    AraTrust is a comprehensive Trustworthiness benchmark for LLMs in Arabic.
+    AraTrust comprises 522 human-written multiple-choice questions addressing
+    diverse dimensions related to truthfulness, ethics, safety, physical health,
+    mental health, unfairness, illegal activities, privacy, and offensive language.
+
+    - https://huggingface.co/datasets/asas-ai/AraTrust
+    - https://arxiv.org/abs/2403.09017
+
+    Citation:
+
+    ```
+    @misc{alghamdi2024aratrustevaluationtrustworthinessllms,
+      title={AraTrust: An Evaluation of Trustworthiness for LLMs in Arabic},
+      author={Emad A. Alghamdi and Reem I. Masoud and Deema Alnuhait and Afnan Y. Alomairi and Ahmed Ashraf and Mohamed Zaytoon},
+      year={2024},
+      eprint={2403.09017},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2403.09017},
+    }
+    ```
+    """  # noqa: E501
+
+    name = "aratrust"
+    description = "aratrust"
+    tags = ["trustworthiness"]
+
+    OPTION_KEYS = ["A", "B", "C"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset: datasets.Dataset = datasets.load_dataset(
+            "asas-ai/AraTrust",
+            revision="d4dd124ed5b90aeb65a7dda7d88e34fb464a31ec",
+            cache_dir=cache_dir,
+            split="test",
+        )
+        instances: List[Instance] = []
+        for row_index, row in enumerate(dataset):
+            question_text = row["Question"]
+            option_texts = [row[option_key] for option_key in self.OPTION_KEYS if row[option_key]]
+            joined_option_texts = "\n".join(option_texts)
+            input = Input(text=f"{question_text}\n\n{joined_option_texts}\n")
+            references = [Reference(output=Output(text=row["Answer"]), tags=[CORRECT_TAG])]
+            instance = Instance(
+                id=f"id{row_index}",
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/audio_language/casual_conversations2_scenario.py

@@ -19,7 +19,7 @@ from helm.common.audio_utils import extract_audio
 
 
 class CasualConversations2Scenario(Scenario):
-    """
+    r"""
     Casual Conversation v2 (Porgali et al, 2023) is composed of over 5,567 participants (26,467 videos).
     The videos feature paid individuals who agreed to participate in the project and explicitly provided
     Age, Gender, Language/Dialect, Geo-location, Disability, Physical adornments, Physical attributes labels

helm/benchmark/scenarios/audio_language/mustard_scenario.py

@@ -19,7 +19,7 @@ from helm.common.general import ensure_directory_exists, ensure_file_downloaded
 
 
 class MUStARDScenario(Scenario):
-    """
+    r"""
     MUStARD: Multimodal Sarcasm Detection Dataset
 
     A multimodal video corpus for research in automated sarcasm discovery. The dataset is compiled from popular

helm/benchmark/scenarios/audio_language/{ultra_suite_asr_classification.py → ultra_suite_asr_classification_scenario.py}

@@ -14,7 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
 )
 from helm.common.media_object import MediaObject, MultimediaObject
-from helm.common.general import ensure_file_downloaded
+from huggingface_hub import snapshot_download
 
 
 def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
@@ -58,7 +58,6 @@ class UltraSuiteASRClassificationScenario(Scenario):
     name = "speech_disorder"
     description = "A scenario for evaluating speech disorders in children"
     tags = ["audio", "classification", "speech_disorder", "asr"]
-    HF_MAPPING_URL = "https://https://huggingface.co/datasets/SAA-Lab/SLPHelmUltraSuite"
 
     # Classification options
     options: List[str] = ["Healthy", "Unhealthy"]
@@ -70,14 +69,18 @@ class UltraSuiteASRClassificationScenario(Scenario):
         - Audio files (e.g., .mp3)
         - A JSON file with annotations containing 'answer' field
         """
-        print(f"Downloading dataset from {UltraSuiteASRClassificationScenario.HF_MAPPING_URL} to {output_path}")
-        ensure_file_downloaded(source_url=UltraSuiteASRClassificationScenario.HF_MAPPING_URL, target_path=output_path)
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
 
         instances: List[Instance] = []
         split: str = TEST_SPLIT
 
         # Find all pairs of audio and JSON files
-        pairs = find_audio_json_pairs(output_path)
+        pairs = find_audio_json_pairs(data_path)
 
         for audio_path, json_path in tqdm(pairs):
 
@@ -88,9 +91,7 @@ class UltraSuiteASRClassificationScenario(Scenario):
             # Get the correct answer and convert to label
             answer = annotation["disorder_class"]
             # Create references for each option
-            references: List[Reference] = []
-            reference = Reference(Output(text=answer), tags=[CORRECT_TAG])
-            references.append(reference)
+            references: List[Reference] = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
 
             # Create the input with audio and instruction
             content = [