crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/run_specs/multilingual_run_specs.py

@@ -0,0 +1,50 @@
+from helm.benchmark.adaptation.adapter_spec import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("mmmlu")
+def get_mmmlu_spec(locale: str, subject: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.mmmlu_scenario.MMMLUScenario", args={"locale": locale, "subject": subject}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions=f"The following are multiple choice questions (with answers) about {subject.replace('_', ' ')}. Answer the last question. Respond only with only a single letter corresponding to your choice.",  # noqa: E501
+        input_noun="Question",
+        output_noun="Answer",
+    )
+
+    return RunSpec(
+        name=f"mmmlu:locale={locale},subject={subject}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["mmmlu", f"mmmlu_{locale}_{subject}"],
+    )
+
+
+@run_spec_function("exams_multilingual")
+def get_exams_multilingual_spec(language: str, subject: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.exams_multilingual_scenario.EXAMSMultilingualScenario",
+        args={"language": language, "subject": subject},
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions=f"The following are multiple choice questions (with answers) about {subject.replace('_', ' ')}. Answer the last question. Respond only with only a single letter corresponding to your choice.",  # noqa: E501
+        input_noun="Question",
+        output_noun="Answer",
+    )
+
+    return RunSpec(
+        name=f"exams_multilingual:locale={language},subject={subject}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["exams_multilingual", f"exams_multilingual_{language}_{subject}"],
+    )

helm/benchmark/run_specs/speech_disorder_audio_run_specs.py

@@ -73,9 +73,7 @@ def get_ultra_suite_classification_run_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.audio_language.ultra_suite_classification_scenario.UltraSuiteClassificationScenario",  # noqa: E501
     )
-    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
-        input_noun=None, output_noun="Answer", max_train_instances=0
-    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(input_noun=None, output_noun="Answer")
     metric_specs: List[MetricSpec] = audio_classification_metric_specs()
     run_spec_name: str = "ultra_suite_classification"
     return RunSpec(
@@ -92,9 +90,7 @@ def get_ultra_suite_disorder_breakdown_run_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.audio_language.ultra_suite_disorder_breakdown_scenario.UltraSuiteDisorderBreakdownScenario",  # noqa: E501
     )
-    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
-        input_noun=None, output_noun="Answer", max_train_instances=0
-    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(input_noun=None, output_noun="Answer")
     metric_specs: List[MetricSpec] = audio_classification_metric_specs()
     run_spec_name: str = "ultra_suite_classification_breakdown"
     return RunSpec(
@@ -112,13 +108,17 @@ def get_ultra_suite_disorder_breakdown_run_spec() -> RunSpec:
 @run_spec_function("ultra_suite_asr_classification")
 def get_ultra_suite_asr_classification_run_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_asr_classification.UltraSuiteASRClassificationScenario",  # noqa: E501
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_asr_classification_scenario.UltraSuiteASRClassificationScenario",  # noqa: E501
     )
     adapter_spec = _get_generation_adapter_spec(
         instructions="""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording is provided to you, typically consisting of a speech prompt from a pathologist followed by a child's repetition. Based on your expertise transcribe the child's speech into text. Do not make any assumptions about the words the child is expected to say. Only transcribe based on the words that the child actually says. Only respond with the text transcription, no other text or commentary.""",  # noqa: E501
-        max_tokens=10,
+        max_tokens=50,
     )
-    metric_specs: List[MetricSpec] = audio_classification_metric_specs()
+    metric_specs: List[MetricSpec] = [
+        MetricSpec(
+            class_name="helm.benchmark.metrics.ultra_suite_asr_classification_metrics.UltraSuiteASRMetric", args={}
+        )
+    ]
     run_spec_name: str = "ultra_suite_asr_classification"
     return RunSpec(
         name=run_spec_name,
@@ -133,7 +133,7 @@ def get_ultra_suite_asr_classification_run_spec() -> RunSpec:
 @run_spec_function("ultra_suite_asr_transcription")
 def get_ultra_suite_asr_transcription_run_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_asr_classification.UltraSuiteASRClassificationScenario",  # noqa: E501
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_asr_transcription_scenario.UltraSuiteASRTranscriptionScenario",  # noqa: E501
     )
     adapter_spec = _get_generation_adapter_spec(
         instructions="""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. Based on your expertise transcribe the child's speech into text. Try to understand what the child is expected to say. And only respond with the transcription of the child's speech. Not the pathologist's prompt or any other commentary. Only respond with the text transcription, no other text, commentary or punctuations.""",  # noqa: E501
@@ -155,9 +155,7 @@ def get_ultra_suite_disorder_symptoms_run_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.audio_language.ultra_suite_disorder_symptoms_scenario.UltraSuiteDisorderSymptomsScenario",  # noqa: E501
     )
-    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
-        input_noun=None, output_noun="Answer", max_train_instances=0
-    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(input_noun=None, output_noun="Answer")
     metric_specs: List[MetricSpec] = audio_classification_metric_specs()
     run_spec_name: str = "ultra_suite_disorder_symptoms"
     return RunSpec(

helm/benchmark/runner.py

@@ -39,10 +39,15 @@ from helm.benchmark.window_services.tokenizer_service import TokenizerService
 
 
 LATEST_SYMLINK: str = "latest"
+_CURRENT_RUN_SPEC_NAME: typing.Optional[str] = None
 _BENCHMARK_OUTPUT_PATH: str = "benchmark_output"
 _CACHED_MODELS_FOLDER: str = "models"
 
 
+def _get_current_run_spec_name() -> typing.Optional[str]:
+    return _CURRENT_RUN_SPEC_NAME
+
+
 def get_benchmark_output_path() -> str:
     """Get the benchmark output path.
 
@@ -225,6 +230,8 @@ class Runner:
             raise RunnerError(f"Failed runs: [{failed_runs_str}]")
 
     def run_one(self, run_spec: RunSpec):
+        global _CURRENT_RUN_SPEC_NAME
+        _CURRENT_RUN_SPEC_NAME = run_spec.name
         run_path: str = self._get_run_path(run_spec)
         if self.skip_completed_runs and self._is_run_completed(run_path):
             hlog(f"Skipping run {run_spec.name} because run is completed and all output files exist.")

helm/benchmark/scenarios/aci_bench_scenario.py

@@ -1,6 +1,7 @@
 import json
 import os
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -10,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_file_downloaded
 
@@ -124,3 +126,24 @@ class ACIBenchScenario(Scenario):
             instances.extend(self.process_json(test_json, TEST_SPLIT))
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="aci_bench",
+            display_name="ACI-Bench",
+            description="ACI-Bench is a benchmark of real-world patient-doctor conversations paired "
+            "with structured clinical notes. The benchmark evaluates a model's ability to "
+            "understand spoken medical dialogue and convert it into formal clinical "
+            "documentation, covering sections such as history of present illness, physical "
+            "exam findings, results, and assessment and plan [(Yim et al., "
+            "2024)](https://www.nature.com/articles/s41597-023-02487-3).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Extract and structure information from patient-doctor " "conversations",
+                when="Any",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="aci_bench_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/air_bench_scenario.py

@@ -2,6 +2,7 @@ import datasets
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -9,6 +10,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -53,3 +55,22 @@ class AIRBench2024Scenario(Scenario):
             instance = Instance(input=input, references=references, split=TEST_SPLIT)
             instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="air_bench_2024",
+            display_name="AIRBench 2024",
+            description="AIRBench 2024 is a AI safety benchmark that aligns with emerging government "
+            "regulations and company policies. It consists of diverse, malicious prompts "
+            "spanning categories of the regulation-based safety categories in the AIR 2024 "
+            "safety taxonomy.\n",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction-following text generation",
+                what="malicious prompts",
+                when="2024",
+                who="dataset authors and language models",
+                language="English",
+            ),
+            main_metric="air_score",
+            main_split="test",
+        )

helm/benchmark/scenarios/alghafa_scenario.py

@@ -0,0 +1,126 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class AlGhafaScenario(Scenario):
+    """AlGhafa Evaluation Benchmark for Arabic Language Models
+
+    EXPERIMENTAL: This scenario may have future reverse incompatible changes.
+
+    Multiple-choice evaluation benchmark for zero- and few-shot evaluation of Arabic LLMs,
+    consisting of
+
+    - https://huggingface.co/datasets/OALL/AlGhafa-Arabic-LLM-Benchmark-Native/
+    - https://aclanthology.org/2023.arabicnlp-1.21/
+
+    Citation:
+
+    ```
+    @inproceedings{almazrouei-etal-2023-alghafa,
+        title = "{A}l{G}hafa Evaluation Benchmark for {A}rabic Language Models",
+        author = "Almazrouei, Ebtesam  and
+        Cojocaru, Ruxandra  and
+        Baldo, Michele  and
+        Malartic, Quentin  and
+        Alobeidli, Hamza  and
+        Mazzotta, Daniele  and
+        Penedo, Guilherme  and
+        Campesan, Giulia  and
+        Farooq, Mugariya  and
+        Alhammadi, Maitha  and
+        Launay, Julien  and
+        Noune, Badreddine",
+        editor = "Sawaf, Hassan  and
+        El-Beltagy, Samhaa  and
+        Zaghouani, Wajdi  and
+        Magdy, Walid  and
+        Abdelali, Ahmed  and
+        Tomeh, Nadi  and
+        Abu Farha, Ibrahim  and
+        Habash, Nizar  and
+        Khalifa, Salam  and
+        Keleg, Amr  and
+        Haddad, Hatem  and
+        Zitouni, Imed  and
+        Mrini, Khalil  and
+        Almatham, Rawan",
+        booktitle = "Proceedings of ArabicNLP 2023",
+        month = dec,
+        year = "2023",
+        address = "Singapore (Hybrid)",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2023.arabicnlp-1.21/",
+        doi = "10.18653/v1/2023.arabicnlp-1.21",
+        pages = "244--275",
+        abstract = "Recent advances in the space of Arabic large language models have opened up a wealth of potential practical applications. From optimal training strategies, large scale data acquisition and continuously increasing NLP resources, the Arabic LLM landscape has improved in a very short span of time, despite being plagued by training data scarcity and limited evaluation resources compared to English. In line with contributing towards this ever-growing field, we introduce AlGhafa, a new multiple-choice evaluation benchmark for Arabic LLMs. For showcasing purposes, we train a new suite of models, including a 14 billion parameter model, the largest monolingual Arabic decoder-only model to date. We use a collection of publicly available datasets, as well as a newly introduced HandMade dataset consisting of 8 billion tokens. Finally, we explore the quantitative and qualitative toxicity of several Arabic models, comparing our models to existing public Arabic LLMs."
+    }
+    ```
+    """  # noqa: E501
+
+    name = "alghafa"
+    description = "AlGhafa"
+    tags = ["multiple choice"]
+
+    HF_SPLIT_TO_HELM_SPLIT = {"validation": TRAIN_SPLIT, "test": TEST_SPLIT}
+    REFERENCE_PREFIX = "sol"
+
+    def __init__(self, subset: str):
+        super().__init__()
+        self.subset = subset
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
+            self.subset,
+            revision="a31ebd34ca311d7e0cfc6ad7f458b3435af280f5",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            option_indexes = [
+                int(s.removeprefix(self.REFERENCE_PREFIX))
+                for s in dataset[0].keys()
+                if s.startswith(self.REFERENCE_PREFIX)
+            ]
+            for row_index, row in enumerate(dataset):
+                input = Input(text=row["query"])
+                references: List[Reference] = []
+                # Need to add 1 because label is zero-indexed and has a value from 0 to (N - 1),
+                # but column names are 1 indexed and have values from "sol1" to "solN"
+                correct_option_index = int(row["label"]) + 1
+                for option_index in option_indexes:
+                    column_name = f"{self.REFERENCE_PREFIX}{option_index}"
+                    references.append(
+                        Reference(
+                            output=Output(text=row[column_name]),
+                            tags=[CORRECT_TAG] if option_index == correct_option_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=f"id{row_index}_{split_name}",
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/alrage_scenario.py

@@ -0,0 +1,54 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class ALRAGEScenario(Scenario):
+    """ALRAGE"""  # noqa: E501
+
+    name = "alrage"
+    description = "ALRAGE"
+    tags = ["open-book question answering"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset: datasets.Dataset = datasets.load_dataset(
+            "OALL/ALRAGE",
+            revision="4827b2ed2436aea578e84d9bd4150b66ab8bbe0e",
+            split="train",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=f"السؤال:\n{row['question']}\n\nالسياقات المقترحة:\n{row['candidates']}\n")
+            references: List[Reference] = []
+            references = [
+                Reference(
+                    output=Output(text=row["gold_answer"]),
+                    tags=[CORRECT_TAG],
+                )
+            ]
+            instance = Instance(
+                id=row["id"],
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/anthropic_hh_rlhf_scenario.py

@@ -2,7 +2,8 @@ import re
 from typing import List, Any, Dict
 from datasets import load_dataset
 
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TRAIN_SPLIT, TEST_SPLIT
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TRAIN_SPLIT, TEST_SPLIT, ScenarioMetadata
 
 
 class AnthropicHHRLHFScenario(Scenario):
@@ -88,3 +89,24 @@ class AnthropicHHRLHFScenario(Scenario):
                 )
                 instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="anthropic_hh_rlhf",
+            display_name="Anthropic RLHF dataset",
+            short_display_name="Anthropic RLHF dataset",
+            description="The dialogue datasets released by Anthropic to facilitate research in model "
+            "helpfulness and harmlessness ([Bai et al., "
+            "2022](https://arxiv.org/pdf/2204.05862.pdf); [Ganguli et al., "
+            "2022](https://arxiv.org/pdf/2209.07858.pdf)). We only use the first utterance "
+            "of each dialogue.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Human-LM dialogues and preference labels",
+                when="2022",
+                who="Workers from MTurk and Upwork, language models from Anthropic",
+                language="English",
+            ),
+            main_metric="Helpfulness",
+            main_split="test",
+        )

helm/benchmark/scenarios/anthropic_red_team_scenario.py

@@ -2,7 +2,8 @@ import re
 from typing import List, Any, Dict
 from datasets import load_dataset
 
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TRAIN_SPLIT, TEST_SPLIT
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TRAIN_SPLIT, TEST_SPLIT, ScenarioMetadata
 
 
 class AnthropicRedTeamScenario(Scenario):
@@ -69,3 +70,13 @@ class AnthropicRedTeamScenario(Scenario):
                 )
                 instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="anthropic_red_team",
+            display_name="Anthropic Red Team",
+            description="Anthropic Red Team",
+            taxonomy=TaxonomyInfo(task="instruction following sfaety", what="?", when="?", who="?", language="English"),
+            main_metric="safety_score",
+            main_split="test",
+        )

helm/benchmark/scenarios/arabic_exams_scenario.py

@@ -0,0 +1,114 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.hierarchical_logger import hwarn
+
+
+class ArabicEXAMSScenario(Scenario):
+    """The Arabic subset of the EXAMS High School Examinations Dataset for Multilingual Question Answering
+
+    We use the Open Arabic LLM Leaderboard (OALL) version mirror of the Arabic subset of EXAMS, which is in-turn based
+    on the AceGPT version.
+
+    See: https://www.tii.ae/news/introducing-open-arabic-llm-leaderboard-empowering-arabic-language-modeling-community
+
+    References:
+
+    ```
+    @misc{huang2024acegptlocalizinglargelanguage,
+        title={AceGPT, Localizing Large Language Models in Arabic},
+        author={Huang Huang and Fei Yu and Jianqing Zhu and Xuening Sun and Hao Cheng and Dingjie Song and Zhihong Chen and Abdulmohsen Alharthi and Bang An and Juncai He and Ziche Liu and Zhiyi Zhang and Junying Chen and Jianquan Li and Benyou Wang and Lian Zhang and Ruoyu Sun and Xiang Wan and Haizhou Li and Jinchao Xu},
+        year={2024},
+        eprint={2309.12053},
+        archivePrefix={arXiv},
+        primaryClass={cs.CL},
+        url={https://arxiv.org/abs/2309.12053},
+    }```
+
+    ```
+    @inproceedings{hardalov-etal-2020-exams,
+        title = "{EXAMS}: A Multi-subject High School Examinations Dataset for Cross-lingual and Multilingual Question Answering",
+        author = "Hardalov, Momchil  and
+        Mihaylov, Todor  and
+        Zlatkova, Dimitrina  and
+        Dinkov, Yoan  and
+        Koychev, Ivan  and
+        Nakov, Preslav",
+        editor = "Webber, Bonnie  and
+        Cohn, Trevor  and
+        He, Yulan  and
+        Liu, Yang",
+        booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
+        month = nov,
+        year = "2020",
+        address = "Online",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2020.emnlp-main.438/",
+        doi = "10.18653/v1/2020.emnlp-main.438",
+        pages = "5427--5444",
+        abstract = "We propose EXAMS {--} a new benchmark dataset for cross-lingual and multilingual question answering for high school examinations. We collected more than 24,000 high-quality high school exam questions in 16 languages, covering 8 language families and 24 school subjects from Natural Sciences and Social Sciences, among others.EXAMS offers unique fine-grained evaluation framework across multiple languages and subjects, which allows precise analysis and comparison of the proposed models. We perform various experiments with existing top-performing multilingual pre-trained models and show that EXAMS offers multiple challenges that require multilingual knowledge and reasoning in multiple domains. We hope that EXAMS will enable researchers to explore challenging reasoning and knowledge transfer methods and pre-trained models for school question answering in various languages which was not possible by now. The data, code, pre-trained models, and evaluation are available at http://github.com/mhardalov/exams-qa."
+    }```
+    """  # noqa: E501
+
+    name = "arabic_exams"
+    description = "EXAMS is a benchmark dataset for multilingual and cross-lingual question answering from high school examinations. "  # noqa: E501
+    tags = ["knowledge", "multiple_choice"]
+
+    CHOICES = ["A", "B", "C", "D"]
+    # Remap validation split to train split
+    HF_SPLIT_TO_HELM_SPLIT = {"validation": TRAIN_SPLIT, "test": TEST_SPLIT}
+
+    def __init__(self, subject: str):
+        super().__init__()
+        self.subject: str = subject.replace("_", " ")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits = datasets.load_dataset(
+            "OALL/Arabic_EXAMS",
+            revision="bc7a29346dbcaa16a8cd883b1f3e681ab2b7ff2a",
+            cache_dir=cache_dir,
+        )
+
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            for row in dataset:
+                subject = row["id"].split("-")[0]
+                if self.subject != "all" and self.subject != subject:
+                    continue
+                input = Input(text=row["question"])
+                references: List[Reference] = []
+                if row["answer"] not in self.CHOICES:
+                    hwarn(f"Invalid value in answer column in row: {row}")
+                    continue
+                correct_choice = row["answer"]
+                for choice in self.CHOICES:
+                    references.append(
+                        Reference(
+                            output=Output(text=row[choice]),
+                            tags=[CORRECT_TAG] if choice == correct_choice else [],
+                        )
+                    )
+                instance = Instance(
+                    id=row["id"],
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/arabic_mmlu_scenario.py

@@ -0,0 +1,82 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class ArabicMMLUScenario(Scenario):
+    """ArabicMMLU
+
+    ArabicMMLU is the first multi-task language understanding benchmark
+    for Arabic language, sourced from school exams across diverse educational
+    levels in different countries spanning North Africa, the Levant, and the
+    Gulf regions. The data comprises 40 tasks and 14,575 multiple-choice questions
+    in Modern Standard Arabic (MSA), and is carefully constructed by collaborating
+    with native speakers in the region.
+
+    - https://huggingface.co/datasets/MBZUAI/ArabicMMLU
+    - https://aclanthology.org/2024.findings-acl.334/
+    """
+
+    name = "arabic_mmlu"
+    description = "Arabic Massive Multitask Language Understanding"
+    tags = ["knowledge", "multiple_choice"]
+
+    OPTIONS = ["A", "B", "C", "D"]
+    HF_SPLIT_TO_HELM_SPLIT = {"dev": TRAIN_SPLIT, "test": TEST_SPLIT}
+
+    def __init__(self, subset: str):
+        super().__init__()
+        self.subset = subset.replace("_", " ")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "MBZUAI/ArabicMMLU",
+            self.subset,
+            revision="7aa530e2893ac420352b3f5c1a1310c010e9758b",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            for row_index, row in enumerate(dataset):
+                input = Input(text=row["Question"])
+                references: List[Reference] = []
+                correct_option_index = ord(row["Answer Key"]) - ord("A") + 1
+                for option_index in range(1, 6):
+                    column_name = f"Option {option_index}"
+                    if not row[column_name]:
+                        continue
+                    references.append(
+                        Reference(
+                            # Need to convert column to string because the references are floats
+                            # for the subject "Math (Primary School)"
+                            output=Output(text=str(row[column_name])),
+                            tags=[CORRECT_TAG] if option_index == correct_option_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=f"id{row_index}",
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances