crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/run_specs/winogrande_afr_run_specs.py

@@ -0,0 +1,47 @@
+"""Run spec functions for Winogrande human-translated into 11 African languages
+
+Available langs: "af", "zu", "xh", "am", "bm", "ig", "nso", "sn", "st", "tn", "ts" (see lang_map below for language code mapping to language name, or here for ISO code reference: https://huggingface.co/languages)
+"""  # noqa: E501
+
+from helm.benchmark.adaptation.adapter_spec import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("winogrande_afr")
+def get_winogrande_afr_spec(lang: str, method: str = ADAPT_MULTIPLE_CHOICE_JOINT) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.winogrande_afr_scenario.Winogrande_Afr_Scenario", args={"lang": lang}
+    )
+
+    lang_map = {
+        "af": "Afrikaans",
+        "zu": "Zulu",
+        "xh": "Xhosa",
+        "am": "Amharic",
+        "bm": "Bambara",
+        "ig": "Igbo",
+        "nso": "Sepedi",
+        "sn": "Shona",
+        "st": "Sesotho",
+        "tn": "Setswana",
+        "ts": "Tsonga",
+    }
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=method,
+        instructions=f"The following are binary choice fill-in-the-blank sentences (with answers), "
+        f"requiring common sense reasoning in {lang_map[lang]}.",
+        input_noun="Question",
+        output_noun="Answer",
+    )
+
+    return RunSpec(
+        name=f"winogrande_afr:lang={lang},method={method}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["winogrande_afr", f"winogrande_afr_{lang}"],
+    )

helm/benchmark/runner.py

@@ -6,45 +6,50 @@ import traceback
 import typing
 from collections import Counter
 import dataclasses
-from dataclasses import dataclass, field
 from typing import Any, Dict, List
 import numpy as np
 
 from tqdm import tqdm
 
+from helm.benchmark.adaptation.request_state import RequestState
 from helm.common.general import ensure_directory_exists, write, asdict_without_nones
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hlog, htrack_block, hwarn
 from helm.common.cache import cache_stats
-from .augmentations.data_augmenter import DataAugmenterSpec
-from .scenarios.scenario import (
+from helm.benchmark.scenarios.scenario import (
     EVAL_SPLITS,
     TRAIN_SPLIT,
     Scenario,
-    ScenarioSpec,
     create_scenario,
     Instance,
     get_scenario_cache_path,
     with_instance_ids,
 )
-from .adaptation.adapters.adapter import Adapter
-from .adaptation.adapters.adapter_factory import AdapterFactory
-from .adaptation.scenario_state import ScenarioState
-from .adaptation.adapter_spec import AdapterSpec
-from .data_preprocessor import DataPreprocessor
-from .executor import ExecutionSpec, Executor
-from .metrics.dry_run_metrics import DryRunMetric
-from .metrics.metric_name import MetricName
-from .metrics.metric_service import MetricService
-from .metrics.metric import Metric, MetricSpec, MetricResult, PerInstanceStats, create_metric, Stat
-from .window_services.tokenizer_service import TokenizerService
+from helm.benchmark.adaptation.adapters.adapter import Adapter
+from helm.benchmark.adaptation.adapters.adapter_factory import AdapterFactory
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.run_spec import RunSpec
+from helm.benchmark.data_preprocessor import DataPreprocessor
+from helm.benchmark.executor import ExecutionSpec, Executor
+from helm.benchmark.annotation_executor import AnnotationExecutionSpec, AnnotationExecutor
+from helm.benchmark.metrics.dry_run_metrics import DryRunMetric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats, create_metric, Stat
+from helm.benchmark.window_services.tokenizer_service import TokenizerService
 
 
 LATEST_SYMLINK: str = "latest"
+_CURRENT_RUN_SPEC_NAME: typing.Optional[str] = None
 _BENCHMARK_OUTPUT_PATH: str = "benchmark_output"
+_CACHED_MODELS_FOLDER: str = "models"
+
+
+def _get_current_run_spec_name() -> typing.Optional[str]:
+    return _CURRENT_RUN_SPEC_NAME
 
 
 def get_benchmark_output_path() -> str:
-    """Get the genchmark output path.
+    """Get the benchmark output path.
 
     Many run spec functions need to know the benchmark output path,
     but there is no way to pass it via  the run spec function,
@@ -52,8 +57,15 @@ def get_benchmark_output_path() -> str:
     return _BENCHMARK_OUTPUT_PATH
 
 
+def get_cached_models_path() -> str:
+    """Get the cached models pat within the benchmark output path."""
+    path: str = os.path.join(get_benchmark_output_path(), _CACHED_MODELS_FOLDER)
+    ensure_directory_exists(path)
+    return path
+
+
 def set_benchmark_output_path(benchmark_output_path: str) -> None:
-    """Set the genchmark output path."""
+    """Set the benchmark output path."""
     global _BENCHMARK_OUTPUT_PATH
     _BENCHMARK_OUTPUT_PATH = benchmark_output_path
 
@@ -64,40 +76,6 @@ class RunnerError(Exception):
     pass
 
 
-@dataclass(frozen=True)
-class RunSpec:
-    """
-    Specifies how to do a single run, which gets a scenario, adapts it, and
-    computes a list of stats based on the defined metrics.
-    """
-
-    # Unique identifier of the RunSpec
-    name: str
-
-    # Which scenario
-    scenario_spec: ScenarioSpec
-
-    # Specifies how to adapt an instance into a set of requests
-    adapter_spec: AdapterSpec
-
-    # What to evaluate on
-    metric_specs: List[MetricSpec]
-
-    # Data augmenter. The default `DataAugmenterSpec` does nothing.
-    data_augmenter_spec: DataAugmenterSpec = DataAugmenterSpec()
-
-    # Groups that this run spec belongs to (for aggregation)
-    groups: List[str] = field(default_factory=list)
-
-    def __post_init__(self):
-        """
-        `self.name` is used as the name of the output folder for the `RunSpec`.
-        Clean up `self.name` by replacing any "/"'s with "_".
-        """
-        # TODO: Don't mutate name! clean this up before passing it into the constructor here
-        object.__setattr__(self, "name", self.name.replace(os.path.sep, "_"))
-
-
 def remove_stats_nans(stats: List[Stat]) -> List[Stat]:
     """Return a new list of stats with stats with NaNs removed.
 
@@ -109,7 +87,7 @@ def remove_stats_nans(stats: List[Stat]) -> List[Stat]:
     result: List[Stat] = []
     for stat in stats:
         if math.isnan(stat.sum):
-            hlog(f"WARNING: Removing stat {stat.name.name} because its value is NaN")
+            hwarn(f"Removing stat {stat.name.name} because its value is NaN")
             continue
         result.append(stat)
     return result
@@ -129,7 +107,9 @@ def remove_per_instance_stats_nans(per_instance_stats_list: List[PerInstanceStat
     return result
 
 
-def downsample_eval_instances(instances: List[Instance], max_eval_instances: int) -> List[Instance]:
+def downsample_eval_instances(
+    instances: List[Instance], max_eval_instances: int, eval_splits: List[str]
+) -> List[Instance]:
     """
     Get the instances necessary for this run:
     Train instances (split=train): keep all (if any) for in-context learning
@@ -138,7 +118,7 @@ def downsample_eval_instances(instances: List[Instance], max_eval_instances: int
     """
     all_train_instances: List[Instance] = [instance for instance in instances if instance.split == TRAIN_SPLIT]
 
-    all_eval_instances: List[Instance] = [instance for instance in instances if instance.split in EVAL_SPLITS]
+    all_eval_instances: List[Instance] = [instance for instance in instances if instance.split in eval_splits]
     if len(all_eval_instances) > max_eval_instances:
         # The random sampling includes instances monotonically.
         np.random.seed(0)
@@ -179,9 +159,18 @@ class Runner:
         exit_on_error: bool,
     ):
         self.executor = Executor(execution_spec)
+        self.annotator_executor = AnnotationExecutor(
+            AnnotationExecutionSpec(
+                local_path=execution_spec.local_path if execution_spec.local_path is not None else "",
+                parallelism=execution_spec.parallelism,
+                dry_run=execution_spec.dry_run,
+                sqlite_cache_backend_config=execution_spec.sqlite_cache_backend_config,
+                mongo_cache_backend_config=execution_spec.mongo_cache_backend_config,
+            )
+        )
         self.dry_run: bool = execution_spec.dry_run
-        self.tokenizer_service = TokenizerService(self.executor.service, execution_spec.auth)
-        self.metric_service = MetricService(self.executor.service, execution_spec.auth)
+        self.tokenizer_service = TokenizerService(self.executor.context)
+        self.metric_service = MetricService(self.executor.context)
         self.skip_instances: bool = skip_instances
         self.cache_instances: bool = cache_instances
         self.cache_instances_only: bool = cache_instances_only
@@ -241,6 +230,8 @@ class Runner:
             raise RunnerError(f"Failed runs: [{failed_runs_str}]")
 
     def run_one(self, run_spec: RunSpec):
+        global _CURRENT_RUN_SPEC_NAME
+        _CURRENT_RUN_SPEC_NAME = run_spec.name
         run_path: str = self._get_run_path(run_spec)
         if self.skip_completed_runs and self._is_run_completed(run_path):
             hlog(f"Skipping run {run_spec.name} because run is completed and all output files exist.")
@@ -280,12 +271,14 @@ class Runner:
             return  # Exit after saving the instances.
 
         # Give each instance a unique ID
-        instances = with_instance_ids(instances)
+        if any([instance.id is None for instance in instances]):
+            instances = with_instance_ids(instances)
 
         # Get the instances necessary for this run.
         max_eval_instances = run_spec.adapter_spec.max_eval_instances
+        eval_splits = run_spec.adapter_spec.eval_splits or EVAL_SPLITS
         if max_eval_instances is not None:
-            instances = downsample_eval_instances(instances, max_eval_instances)
+            instances = downsample_eval_instances(instances, max_eval_instances, eval_splits)
 
         # Data preprocessing
         instances = DataPreprocessor(run_spec.data_augmenter_spec).preprocess(
@@ -294,15 +287,23 @@ class Runner:
 
         # Adapt (convert to requests)
         adapter: Adapter = AdapterFactory.get_adapter(run_spec.adapter_spec, self.tokenizer_service)
-        scenario_state: ScenarioState = adapter.adapt(instances, self.executor.execution_spec.parallelism)
+        request_states: List[RequestState] = adapter.adapt(instances, self.executor.execution_spec.parallelism)
+        scenario_state: ScenarioState = ScenarioState(
+            adapter_spec=run_spec.adapter_spec,
+            request_states=request_states,
+            annotator_specs=run_spec.annotators,
+        )
 
         # Execute (fill up results)
         scenario_state = self.executor.execute(scenario_state)
 
+        # Annotate (post-process the results)
+        scenario_state = self.annotator_executor.execute(scenario_state)
+
         # Apply the metrics
         # When performing a dry run, only estimate the number of tokens instead
         # of calculating the metrics.
-        metrics: List[Metric] = (
+        metrics: List[MetricInterface] = (
             [DryRunMetric()] if self.dry_run else [create_metric(metric_spec) for metric_spec in run_spec.metric_specs]
         )
         stats: List[Stat] = []
@@ -324,7 +325,7 @@ class Runner:
         metric_counts: typing.Counter[MetricName] = Counter([stat.name for stat in stats])
         for metric_name, count in metric_counts.items():
             if count > 1:
-                hlog(f"WARNING: duplicate metric name {metric_name}")
+                hwarn(f"duplicate metric name {metric_name}")
 
         # Print out the number of stats
         hlog(f"Generated {len(stats)} stats.")

helm/benchmark/runner_config_registry.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+import dacite
+import yaml
+
+
+@dataclass
+class RunnerConfigSpec:
+    helm_max_concurrent_workers: int = -1
+    slurm_monitor_interval: int = 60
+    slurm_args: Optional[Dict[str, Any]] = None
+
+
+RUNNER_CONFIG = RunnerConfigSpec()
+
+
+def register_runner_config_from_path(dir_path: str) -> None:
+    global RUNNER_CONFIG
+    with open(dir_path, "r") as f:
+        raw = yaml.safe_load(f)
+    RUNNER_CONFIG = dacite.from_dict(RunnerConfigSpec, raw)

helm/benchmark/scenarios/aci_bench_scenario.py

@@ -0,0 +1,149 @@
+import json
+import os
+from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_file_downloaded
+
+
+class ACIBenchScenario(Scenario):
+    """
+    From "Aci-bench: a Novel Ambient Clinical Intelligence Dataset for Benchmarking Automatic Visit Note Generation"
+    (Yim et al.), ACI-Bench is the largest dataset to date tackling the problem of AI-assisted note generation from
+    doctor-patient dialogue. This dataset enables benchmarking and evaluation of generative models, focusing on the
+    arduous task of converting clinical dialogue into structured electronic medical records (EMR).
+
+    Example from the dataset:
+
+    Dialogue:
+    [doctor] hi, brian. how are you?
+    [patient] hi, good to see you.
+    [doctor] it's good to see you too. so, i know the nurse told you a little bit about dax.
+    [patient] mm-hmm.
+    [doctor] i'd like to tell dax about you, okay?
+    [patient] sure.
+
+    Note:
+    CHIEF COMPLAINT
+
+    Follow-up of chronic problems.
+
+    HISTORY OF PRESENT ILLNESS
+
+    @Article{ACI-Bench,
+    author = {Wen-wai Yim, Yujuan Fu, Asma Ben Abacha, Neal Snider, Thomas Lin, Meliha Yetisgen},
+    title = {Aci-bench: a Novel Ambient Clinical Intelligence Dataset for Benchmarking Automatic Visit Note Generation},
+    journal = {Nature Scientific Data},
+    year = {2023},
+    abstract = {Recent immense breakthroughs in generative models have precipitated re-imagined ubiquitous
+    usage of these models in all applications. One area that can benefit by improvements in artificial intelligence (AI)
+    is healthcare. The note generation task from doctor-patient encounters, and its associated electronic medical record
+    documentation, is one of the most arduous time-consuming tasks for physicians. It is also a natural prime potential
+    beneficiary to advances in generative models. However with such advances, benchmarking is more critical than ever.
+    Whether studying model weaknesses or developing new evaluation metrics, shared open datasets are an imperative part
+    of understanding the current state-of-the-art. Unfortunately as clinic encounter conversations are not routinely
+    recorded and are difficult to ethically share due to patient confidentiality, there are no sufficiently large clinic
+    dialogue-note datasets to benchmark this task. Here we present the Ambient Clinical Intelligence Benchmark
+    corpus, the largest dataset to date tackling the problem of AI-assisted note generation from visit dialogue. We also
+    present the benchmark performances of several common state-of-the-art approaches.}}
+
+    Task:
+    Given a doctor-patient dialogue, models must generate a clinical note that summarizes the conversation,
+    focusing on the chief complaint, history of present illness, and other relevant clinical information.
+    """
+
+    PREFIX = (
+        "https://raw.githubusercontent.com/"
+        "wyim/aci-bench/e75b383172195414a7a68843ec4876e83e5409f7/data/challenge_data_json"
+    )
+    TRAIN_URL = f"{PREFIX}/train_full.json"
+    TEST_URLS = [
+        f"{PREFIX}/clinicalnlp_taskB_test1_full.json",
+        f"{PREFIX}/clef_taskC_test3_full.json",
+        f"{PREFIX}/clinicalnlp_taskC_test2_full.json",
+    ]
+
+    name = "aci_bench"
+    description = (
+        "ACI-Bench is a benchmark of real-world patient-doctor conversations paired with"
+        "structured clinical notes. The benchmark evaluates a model's ability to understand"
+        "spoken medical dialogue and convert it into formal clinical documentation, covering"
+        "sections such as history of present illness, physical exam findings, results, and assessment"
+        "and plan."
+    )
+    tags = ["summarization", "medicine"]
+
+    def download_json(self, url: str, output_path: str, file_name: str) -> str:
+        """Download the JSON file and save it to the specified path."""
+        json_path = os.path.join(output_path, file_name)
+        ensure_file_downloaded(source_url=url, target_path=json_path, unpack=False)
+        return json_path
+
+    def process_json(self, json_path: str, split: str) -> List[Instance]:
+        """Read and process the JSON file to generate instances."""
+        instances: List[Instance] = []
+        with open(json_path, "r", encoding="utf-8") as json_file:
+            data = json.load(json_file)
+
+            for entry in data["data"]:
+                dialogue = entry["src"]
+                note = entry["tgt"]
+
+                # Prepare the input text (dialogue)
+                input_text = f"Doctor-patient dialogue:\n\n{dialogue}"
+
+                # Create an instance
+                instance = Instance(
+                    input=Input(text=input_text),
+                    references=[Reference(Output(text=note), tags=[CORRECT_TAG])],
+                    split=split,
+                )
+                instances.append(instance)
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Download and process the dataset to generate instances."""
+        instances: List[Instance] = []
+
+        # Process training set
+        train_json = self.download_json(self.TRAIN_URL, output_path, "aci_bench_train.json")
+        instances.extend(self.process_json(train_json, TRAIN_SPLIT))
+
+        # Process test sets
+        for idx, test_url in enumerate(self.TEST_URLS, start=1):
+            test_json = self.download_json(test_url, output_path, f"aci_bench_test_{idx}.json")
+            instances.extend(self.process_json(test_json, TEST_SPLIT))
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="aci_bench",
+            display_name="ACI-Bench",
+            description="ACI-Bench is a benchmark of real-world patient-doctor conversations paired "
+            "with structured clinical notes. The benchmark evaluates a model's ability to "
+            "understand spoken medical dialogue and convert it into formal clinical "
+            "documentation, covering sections such as history of present illness, physical "
+            "exam findings, results, and assessment and plan [(Yim et al., "
+            "2024)](https://www.nature.com/articles/s41597-023-02487-3).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Extract and structure information from patient-doctor " "conversations",
+                when="Any",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="aci_bench_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/air_bench_scenario.py

@@ -0,0 +1,76 @@
+import datasets
+import os
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class AIRBench2024Scenario(Scenario):
+    """AIRBench 2024
+
+    Pre-publication: References will be added post-publication.
+
+    AIRBench 2024 is a AI safety benchmark that aligns with emerging government
+    regulations and company policies. It consists of 5,619 malicious prompts
+    spanning categories of the regulation-based safety categories in the
+    AIR 2024 safety taxonomy."""
+
+    name = "air_bench_2024"
+    description = (
+        "AIRBench 2024 is a AI safety benchmark that aligns with "
+        "emerging government regulations and company policies"
+    )
+    tags = ["safety"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+
+        # TODO: Switch this to the production dataset when available.
+        dataset = datasets.load_dataset(
+            "stanford-crfm/air-bench-2024",
+            split="test",
+            cache_dir=cache_dir,
+            revision="976d9fbb0f52bce704dcbef5bd7a2652e65f2a21",
+        )
+        instances: List[Instance] = []
+        # TODO: Allow users to filter by category
+        for row in dataset:
+            input = Input(text=row["prompt"])
+            # References are category ID, followed by level 2, 3 and 4 category names.
+            references = [
+                Reference(output=Output(text=row[column_name]), tags=[])
+                for column_name in ["cate-idx", "l2-name", "l3-name", "l4-name"]
+            ]
+            instance = Instance(input=input, references=references, split=TEST_SPLIT)
+            instances.append(instance)
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="air_bench_2024",
+            display_name="AIRBench 2024",
+            description="AIRBench 2024 is a AI safety benchmark that aligns with emerging government "
+            "regulations and company policies. It consists of diverse, malicious prompts "
+            "spanning categories of the regulation-based safety categories in the AIR 2024 "
+            "safety taxonomy.\n",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction-following text generation",
+                what="malicious prompts",
+                when="2024",
+                who="dataset authors and language models",
+                language="English",
+            ),
+            main_metric="air_score",
+            main_split="test",
+        )

helm/benchmark/scenarios/alghafa_scenario.py

@@ -0,0 +1,126 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class AlGhafaScenario(Scenario):
+    """AlGhafa Evaluation Benchmark for Arabic Language Models
+
+    EXPERIMENTAL: This scenario may have future reverse incompatible changes.
+
+    Multiple-choice evaluation benchmark for zero- and few-shot evaluation of Arabic LLMs,
+    consisting of
+
+    - https://huggingface.co/datasets/OALL/AlGhafa-Arabic-LLM-Benchmark-Native/
+    - https://aclanthology.org/2023.arabicnlp-1.21/
+
+    Citation:
+
+    ```
+    @inproceedings{almazrouei-etal-2023-alghafa,
+        title = "{A}l{G}hafa Evaluation Benchmark for {A}rabic Language Models",
+        author = "Almazrouei, Ebtesam  and
+        Cojocaru, Ruxandra  and
+        Baldo, Michele  and
+        Malartic, Quentin  and
+        Alobeidli, Hamza  and
+        Mazzotta, Daniele  and
+        Penedo, Guilherme  and
+        Campesan, Giulia  and
+        Farooq, Mugariya  and
+        Alhammadi, Maitha  and
+        Launay, Julien  and
+        Noune, Badreddine",
+        editor = "Sawaf, Hassan  and
+        El-Beltagy, Samhaa  and
+        Zaghouani, Wajdi  and
+        Magdy, Walid  and
+        Abdelali, Ahmed  and
+        Tomeh, Nadi  and
+        Abu Farha, Ibrahim  and
+        Habash, Nizar  and
+        Khalifa, Salam  and
+        Keleg, Amr  and
+        Haddad, Hatem  and
+        Zitouni, Imed  and
+        Mrini, Khalil  and
+        Almatham, Rawan",
+        booktitle = "Proceedings of ArabicNLP 2023",
+        month = dec,
+        year = "2023",
+        address = "Singapore (Hybrid)",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2023.arabicnlp-1.21/",
+        doi = "10.18653/v1/2023.arabicnlp-1.21",
+        pages = "244--275",
+        abstract = "Recent advances in the space of Arabic large language models have opened up a wealth of potential practical applications. From optimal training strategies, large scale data acquisition and continuously increasing NLP resources, the Arabic LLM landscape has improved in a very short span of time, despite being plagued by training data scarcity and limited evaluation resources compared to English. In line with contributing towards this ever-growing field, we introduce AlGhafa, a new multiple-choice evaluation benchmark for Arabic LLMs. For showcasing purposes, we train a new suite of models, including a 14 billion parameter model, the largest monolingual Arabic decoder-only model to date. We use a collection of publicly available datasets, as well as a newly introduced HandMade dataset consisting of 8 billion tokens. Finally, we explore the quantitative and qualitative toxicity of several Arabic models, comparing our models to existing public Arabic LLMs."
+    }
+    ```
+    """  # noqa: E501
+
+    name = "alghafa"
+    description = "AlGhafa"
+    tags = ["multiple choice"]
+
+    HF_SPLIT_TO_HELM_SPLIT = {"validation": TRAIN_SPLIT, "test": TEST_SPLIT}
+    REFERENCE_PREFIX = "sol"
+
+    def __init__(self, subset: str):
+        super().__init__()
+        self.subset = subset
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
+            self.subset,
+            revision="a31ebd34ca311d7e0cfc6ad7f458b3435af280f5",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            option_indexes = [
+                int(s.removeprefix(self.REFERENCE_PREFIX))
+                for s in dataset[0].keys()
+                if s.startswith(self.REFERENCE_PREFIX)
+            ]
+            for row_index, row in enumerate(dataset):
+                input = Input(text=row["query"])
+                references: List[Reference] = []
+                # Need to add 1 because label is zero-indexed and has a value from 0 to (N - 1),
+                # but column names are 1 indexed and have values from "sol1" to "solN"
+                correct_option_index = int(row["label"]) + 1
+                for option_index in option_indexes:
+                    column_name = f"{self.REFERENCE_PREFIX}{option_index}"
+                    references.append(
+                        Reference(
+                            output=Output(text=row[column_name]),
+                            tags=[CORRECT_TAG] if option_index == correct_option_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=f"id{row_index}_{split_name}",
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances