crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/scenarios/starr_patient_instructions_scenario.py

@@ -1,8 +1,7 @@
-import os
 import csv
 from typing import List
 
-from helm.common.general import ensure_directory_exists
+from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -40,19 +39,27 @@ class StarrPatientInstructionsScenario(Scenario):
     """
 
     name = "starr_patient_instructions"
-    description = "A dataset containing case details used to generate customized post-procedure patient instructions."
+    description = (
+        "PatientInstruct is a benchmark designed to evaluate models on generating personalized"
+        "post-procedure instructions for patients. It includes real-world patient History & Physical"
+        "Note (H&P) and operative report, from which models must produce clear, actionable instructions"
+        "appropriate for patients recovering from medical interventions."
+    )
     tags = ["patient_communication", "healthcare", "instruction_generation", "surgery"]
 
-    def get_instances(self, output_path: str) -> List[Instance]:
-        csv_path = "/share/pi/nigam/suhana/medhelm/data/starr-omop-personalized-care-instr/dataset_cases_qc.csv"
-        # Ensure the directory for the CSV file exists.
-        ensure_directory_exists(os.path.dirname(csv_path))
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
 
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(
+            self.data_path, msg=f"[StarrPatientInstructiosScenario] Required data file not found: '{self.data_path}'"
+        )
         instances: List[Instance] = []
         # For now, we assign all instances to the test split (zero-shot setting).
         split = TEST_SPLIT
 
-        with open(csv_path, "r", encoding="utf-8") as csvfile:
+        with open(self.data_path, "r", encoding="utf-8") as csvfile:
             reader = csv.DictReader(csvfile)
             for row in reader:
                 # Retrieve and strip the relevant fields.

helm/benchmark/scenarios/test_alghafa_scenario.py

@@ -0,0 +1,29 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.alghafa_scenario import AlGhafaScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input
+
+
+@pytest.mark.scenarios
+def test_alghafa_scenario_get_instances():
+    scenario = AlGhafaScenario(subset="mcq_exams_test_ar")
+    with TemporaryDirectory() as tmpdir:
+        actual_instances = scenario.get_instances(tmpdir)
+    assert len(actual_instances) == 562
+    assert actual_instances[0].id == "id0_test"
+    assert actual_instances[0].input == Input(
+        text=(
+            'قال علي بن أبي طالب رضي الله عنه عن عمر بن الخطاب رضي الله عنه " إن كنا لنرى إن في القرآن كلاماً من كلامه ورأياً من رأيه " دلت هذه العبارة على سمة وصفة من صفات عمر بن الخطاب رضي الله عنه هي'  # noqa: E501
+        )
+    )
+    assert len(actual_instances[0].references) == 4
+    assert actual_instances[0].references[0].output.text == "الشجاعة"
+    assert actual_instances[0].references[0].tags == []
+    assert actual_instances[0].references[1].output.text == "نزل القرآن الكريم موافقاً لرأيه في عدة مواضع"
+    assert actual_instances[0].references[1].tags == [CORRECT_TAG]
+    assert actual_instances[0].references[2].output.text == "الشدة في الحق مع اللين والرحمة ."
+    assert actual_instances[0].references[2].tags == []
+    assert actual_instances[0].references[3].output.text == "التواضع"
+    assert actual_instances[0].references[3].tags == []
+    assert actual_instances[0].split == "test"

helm/benchmark/scenarios/test_aratrust_scenario.py

@@ -0,0 +1,21 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.aratrust_scenario import AraTrustScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input
+
+
+@pytest.mark.scenarios
+def test_aratrust_get_instances():
+    scenario = AraTrustScenario()
+    with TemporaryDirectory() as tmpdir:
+        actual_instances = scenario.get_instances(tmpdir)
+    assert len(actual_instances) == 522
+    assert actual_instances[0].id == "id0"
+    assert actual_instances[0].input == Input(
+        text=("حيوان لا يبيض ولا يلد ما هو؟\n\nأ) أنثى الحيوان\n ب) ذكر الحيوان\n ج) كل ما سبق\n")
+    )
+    assert len(actual_instances[0].references) == 1
+    assert actual_instances[0].references[0].output.text == "ب"
+    assert actual_instances[0].references[0].tags == [CORRECT_TAG]
+    assert actual_instances[0].split == "test"

helm/benchmark/scenarios/test_bluex_scenario.py

@@ -0,0 +1,59 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.bluex_scenario import BLUEX_Scenario
+from helm.benchmark.scenarios.scenario import TEST_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_bluex_scenario():
+    scenario = BLUEX_Scenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+
+    assert len(instances) > 100
+
+    assert instances[100].split == TEST_SPLIT
+
+    assert instances[0].input.text.startswith("Rubião fitava a enseada, - eram oito horas da manhã Quem o visse")
+
+    assert len(instances[0].input.text) == 1011
+
+    assert instances[0].references == [
+        Reference(
+            output=Output(
+                text='a contemplação das paisagens naturais, como se lê em "ele admirava aquele pedaço de água quieta".'
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text='a presença de um narrador-personagem, como se lê em "em verdade vos digo que pensava em '
+                'outra coisa".'
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text='a sobriedade do protagonista ao avaliar o seu percurso, como se lê em "Cotejava o passado com '
+                "o presente."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text='o sentido místico e fatalista que rege os destinos, como se lê em "Deus escreve direito por '
+                'linhas tortas".'
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text='a reversibilidade entre o cômico e o trágico, como se lê em "de modo que o que parecia uma '
+                'desgraça...".'
+            ),
+            tags=[CORRECT_TAG],
+        ),
+    ]
+
+    assert instances[0].references[4].is_correct

helm/benchmark/scenarios/test_exams_multilingual_scenario.py

@@ -0,0 +1,29 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.exams_multilingual_scenario import EXAMSMultilingualScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, TRAIN_SPLIT, Input
+
+
+@pytest.mark.scenarios
+def test_exam_multilingual_scenario_get_instances():
+    scenario = EXAMSMultilingualScenario(language="Bulgarian", subject="Physics")
+    with TemporaryDirectory() as tmpdir:
+        actual_instances = scenario.get_instances(tmpdir)
+    assert len(actual_instances) == 393
+    assert actual_instances[0].id == "4c05bbb8-7729-11ea-9116-54bef70b159e"
+    assert actual_instances[0].input == Input(text="Наелектризирането по индукция се обяснява с: ")
+    assert len(actual_instances[0].references) == 4
+    assert actual_instances[0].references[0].output.text == "преразпределение на положителните йони в тялото"
+    assert actual_instances[0].references[0].tags == []
+    assert (
+        actual_instances[0].references[1].output.text == "предаване на електрони от неутрално на наелектризирано тяло"
+    )
+    assert actual_instances[0].references[1].tags == []
+    assert (
+        actual_instances[0].references[2].output.text == "предаване на електрони от наелектризирано на неутрално тяло"
+    )
+    assert actual_instances[0].references[2].tags == []
+    assert actual_instances[0].references[3].output.text == "преразпределение на свободните електрони в тялото"
+    assert actual_instances[0].references[3].tags == [CORRECT_TAG]
+    assert actual_instances[0].split == TRAIN_SPLIT

helm/benchmark/scenarios/test_healtha_br_scenario.py

@@ -0,0 +1,57 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.healthqa_br_scenario import HEALTHQA_BR_Scenario
+from helm.benchmark.scenarios.scenario import TEST_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_healthqa_br_instance():
+    scenario = HEALTHQA_BR_Scenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+
+    instance = instances[35]
+
+    assert instance.split == TEST_SPLIT
+
+    assert instance.input.text.startswith("Homem de 22 anos de idade procura a Unidade Básica")
+
+    assert instance.references == [
+        Reference(
+            output=Output(
+                text="administração de relaxante muscular, colocando o paciente em posição de Trendelenburg, com "
+                "tentativa de redução do volume."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="encaminhamento do paciente ao Serviço de Urgência do Hospital com o pedido de avaliação "
+                "imediata do cirurgião."
+            ),
+            tags=[CORRECT_TAG],
+        ),
+        Reference(
+            output=Output(
+                text="tentativa de redução manual do aumento de volume da região inguinescrotal para a cavidade "
+                "abdominal."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="transiluminação do escroto para tentar diferenciar hérnia inguinal de hidrocele comunicante."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(text="prescrição de antiemético e solicitação de ecografia da região inguinescrotal."),
+            tags=[],
+        ),
+    ]
+
+    correct_refs = [ref for ref in instance.references if CORRECT_TAG in ref.tags]
+    assert len(correct_refs) == 1
+
+    assert instance.references[1].is_correct

helm/benchmark/scenarios/test_infinite_bench_en_qa_scenario.py

@@ -0,0 +1,18 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.infinite_bench_en_qa_scenario import InfiniteBenchEnQAScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+@pytest.mark.scenarios
+def test_infinite_bench_en_qa_scenario():
+    with TemporaryDirectory() as tmpdir:
+        scenario = InfiniteBenchEnQAScenario(max_num_words=10000000)
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 351
+        assert instances[0].split == "test"
+        assert len(instances[0].input.text) == 381829
+        assert len(instances[0].references) == 1
+        assert len(instances[0].references[0].output.text) == 8
+        assert instances[0].references[0].tags == [CORRECT_TAG]

helm/benchmark/scenarios/test_infinite_bench_en_sum_scenario.py

@@ -0,0 +1,31 @@
+import pytest
+import re
+from tempfile import TemporaryDirectory
+from helm.benchmark.scenarios.infinite_bench_en_sum_scenario import InfiniteBenchEnSumScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+def count_words(text: str) -> int:
+    return len(re.split(r"\s+", text.strip()))
+
+
+@pytest.mark.scenarios
+def test_infinite_bench_en_sum_scenario():
+    with TemporaryDirectory() as tmpdir:
+        scenario = InfiniteBenchEnSumScenario(max_num_words=10000000)
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 103
+        assert instances[0].split == "test"
+        assert len(instances[0].input.text) == 1745528
+        references = instances[0].references
+        assert len(references[0].output.text) == 2865
+        assert references[0].tags == [CORRECT_TAG]
+
+        scenario = InfiniteBenchEnSumScenario(max_num_words=100000)
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 48
+        assert instances[0].split == "test"
+        assert len(instances[0].input.text) == 381778
+        references = instances[0].references
+        assert len(references[0].output.text) == 4217
+        assert references[0].tags == [CORRECT_TAG]

helm/benchmark/scenarios/truthful_qa_scenario.py

@@ -77,7 +77,8 @@ class TruthfulQAScenario(Scenario):
         """Downloads the TruthfulQA dataset."""
         # Download the raw data
         data_dir = os.path.join(output_path, "data")
-        url = "https://raw.githubusercontent.com/sylinrl/TruthfulQA/main/TruthfulQA.csv"
+
+        url = "https://raw.githubusercontent.com/sylinrl/TruthfulQA/d71c110897f5d31c5d7f309e7bc316c152f6f031/data/v1/TruthfulQA.csv"  # noqa: E501
         ensure_directory_exists(data_dir)
         ensure_file_downloaded(source_url=url, target_path=os.path.join(data_dir, self.DATASET_FILE_NAME))
 

helm/benchmark/scenarios/vision_language/msr_vtt_scenario.py

@@ -0,0 +1,75 @@
+from collections import defaultdict
+from typing import List
+import json
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+
+
+class MSRVTTScenario(Scenario):
+    """
+    A large-scale video benchmark for video understanding, especially the emerging task of translating video to text.
+    This is achieved by collecting 257 popular queries from a commercial video search engine, with 118 videos for
+    each query. In its current version, MSR-VTT provides 10K web video clips with 41.2 hours and 200K clip-sentence
+    pairs in total, covering the most comprehensive categories and diverse visual content, and representing the
+    largest dataset in terms of sentence and vocabulary. Each clip is annotated with about 20 natural sentences
+    by 1,327 AMT workers.
+
+    Website link: https://cove.thecvf.com/datasets/839
+
+    Citation:
+    MSR-VTT: A Large Video Description Dataset for Bridging Video and Language Jun Xu, Tao Mei, Ting Yao, Yong Rui
+    CVPR 2016
+    """
+
+    DOWNLOAD_URL: str = "https://www.robots.ox.ac.uk/~maxbain/frozen-in-time/data/MSRVTT.zip"
+
+    name = "msr_vtt"
+    description = "Video captioning dataset with 10K web video clips and 200K clip-sentence pairs."
+    tags = ["vision-language", "video", "captioning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the dataset
+        target_path: str = os.path.join(output_path, "data")
+        ensure_file_downloaded(
+            source_url=self.DOWNLOAD_URL,
+            target_path=target_path,
+            unpack=True,
+        )
+
+        annotation_path: str = os.path.join(target_path, "annotation", "MSR_VTT.json")
+        with open(annotation_path, "r") as f:
+            annotations = json.load(f)["annotations"]
+
+        video_path_to_annotations: dict[str, set[str]] = defaultdict(set)
+        for annotation in annotations:
+            video_id: str = annotation["image_id"]
+            video_path: str = os.path.join(target_path, "videos", "all", f"{video_id}.mp4")
+            assert os.path.exists(video_path), f"Video does not exist at path: {video_path}"
+            video_path_to_annotations[video_path].add(annotation["caption"])
+
+        instances: List[Instance] = []
+        for video_path, captions in video_path_to_annotations.items():
+            content: List[MediaObject] = [
+                MediaObject(location=video_path, content_type="video/mp4"),
+            ]
+            references: List[Reference] = [Reference(Output(text=caption), tags=[CORRECT_TAG]) for caption in captions]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=references,
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/server.py

@@ -9,7 +9,7 @@ import json
 from os import path
 import urllib
 
-from bottle import Bottle, static_file, HTTPResponse
+from bottle import Bottle, static_file, HTTPResponse, response
 import yaml
 
 from helm.benchmark.presentation.schema import SCHEMA_CLASSIC_YAML_FILENAME
@@ -21,6 +21,7 @@ app = Bottle()
 
 @app.get("/config.js")
 def serve_config():
+    response.content_type = "application/javascript; charset=UTF-8"
     if app.config["helm.release"]:
         return (
             f'window.BENCHMARK_OUTPUT_BASE_URL = "{app.config["helm.outputurl"]}";\n'

helm/benchmark/slurm_jobs.py

@@ -13,7 +13,6 @@ except ModuleNotFoundError as e:
 
 
 class SlurmJobState:
-    # TODO: Convert to StrEnum after upgrading to Python 3.11
     # Non-exhaustive list of Slurm job states.
     # See: https://slurm.schedmd.com/squeue.html#SECTION_JOB-STATE-CODES
 
@@ -81,7 +80,7 @@ def get_slurm_job_state(job_id: int) -> str:
     except subprocess.CalledProcessError as e:
         # Default CalledProcessError message doesn't have output, so re-raise here to include the output.
         raise Exception(f"{str(e)} output: {e.output}")
-    search_result = re.search("JobState=(\w+)", scontrol_output.decode())
+    search_result = re.search(r"JobState=(\w+)", scontrol_output.decode())
     if not search_result:
         raise Exception(f"Could not extract JobState from scontrol: {scontrol_output.decode()}")
     return search_result.group(1)

helm/benchmark/slurm_runner.py

@@ -26,7 +26,7 @@ from helm.benchmark.slurm_jobs import (
     FAILURE_SLURM_JOB_STATES,
 )
 from helm.common.general import ensure_directory_exists
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hlog, htrack_block, setup_default_logging
 
 from helm.benchmark.runner_config_registry import RUNNER_CONFIG
 
@@ -343,7 +343,14 @@ def main():
         help="Path to the RunSpec JSON file",
         required=True,
     )
+    parser.add_argument(
+        "--log-config",
+        type=str,
+        default=None,
+        help="PATH to a YAML file to customize logging",
+    )
     args = parser.parse_args()
+    setup_default_logging(args.log_config)
 
     # Deserialize SlurmRunner and RunSpec from the given files, then run the RunSpec with the SlurmRunner.
     with open(args.slurm_runner_spec_path, "r") as f:

helm/benchmark/static/schema_arabic.yaml

@@ -0,0 +1,228 @@
+---
+# Schema for Arabic scenarios
+############################################################
+metrics:
+  # Infrastructure metrics:
+  - name: num_perplexity_tokens
+    display_name: '# tokens'
+    description: Average number of tokens in the predicted output (for language modeling, the input too).
+  - name: num_bytes
+    display_name: '# bytes'
+    description: Average number of bytes in the predicted output (for language modeling, the input too).
+
+  - name: num_references
+    display_name: '# ref'
+    description: Number of references.
+  - name: num_train_trials
+    display_name: '# trials'
+    description: Number of trials, where in each trial we choose an independent, random set of training instances.
+  - name: estimated_num_tokens_cost
+    display_name: 'cost'
+    description: An estimate of the number of tokens (including prompt and output completions) needed to perform the request.
+  - name: num_prompt_tokens
+    display_name: '# prompt tokens'
+    description: Number of tokens in the prompt.
+  - name: num_prompt_characters
+    display_name: '# prompt chars'
+    description: Number of characters in the prompt.
+  - name: num_completion_tokens
+    display_name: '# completion tokens'
+    description: Actual number of completion tokens (over all completions).
+  - name: num_output_tokens
+    display_name: '# output tokens'
+    description: Actual number of output tokens.
+  - name: max_num_output_tokens
+    display_name: 'Max output tokens'
+    description: Maximum number of output tokens (overestimate since we might stop earlier due to stop sequences).
+  - name: num_requests
+    display_name: '# requests'
+    description: Number of distinct API requests.
+  - name: num_instances
+    display_name: '# eval'
+    description: Number of evaluation instances.
+  - name: num_train_instances
+    display_name: '# train'
+    description: Number of training instances (e.g., in-context examples).
+  - name: prompt_truncated
+    display_name: truncated
+    description: Fraction of instances where the prompt itself was truncated (implies that there were no in-context examples).
+  - name: finish_reason_length
+    display_name: finish b/c length
+    description: Fraction of instances where the the output was terminated because of the max tokens limit.
+  - name: finish_reason_stop
+    display_name: finish b/c stop
+    description: Fraction of instances where the the output was terminated because of the stop sequences.
+  - name: finish_reason_endoftext
+    display_name: finish b/c endoftext
+    description: Fraction of instances where the the output was terminated because the end of text token was generated.
+  - name: finish_reason_unknown
+    display_name: finish b/c unknown
+    description: Fraction of instances where the the output was terminated for unknown reasons.
+  - name: num_completions
+    display_name: '# completions'
+    description: Number of completions.
+  - name: predicted_index
+    display_name: Predicted index
+    description: Integer index of the reference (0, 1, ...) that was predicted by the model (for multiple-choice).
+  - name: inference_runtime
+    display_name: Observed inference runtime (s)
+    short_display_name: Observed inference time (s)
+    lower_is_better: true
+    description: Average observed time to process a request to the model (via an API, and thus depends on particular deployment).
+
+  # Accuracy metrics:
+  - name: exact_match
+    display_name: Exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_exact_match
+    display_name: Quasi-exact match
+    short_display_name: EM
+    description: Fraction of instances that the predicted output matches a correct reference up to light processing.
+    lower_is_better: false
+  - name: prefix_exact_match
+    display_name: Prefix exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference exactly.
+    lower_is_better: false
+  - name: quasi_prefix_exact_match
+    # TODO: should call this prefix_quasi_exact_match
+    display_name: Prefix quasi-exact match
+    short_display_name: PEM
+    description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
+    lower_is_better: false
+
+############################################################
+perturbations: []
+
+############################################################
+metric_groups:
+  - name: accuracy
+    display_name: Accuracy
+    aggregation_strategies: 
+      - mean
+    metrics:
+      - name: ${main_name}
+        split: ${main_split}
+
+  - name: efficiency
+    display_name: Efficiency
+    aggregation_strategies: 
+      - mean
+    metrics:
+    - name: inference_runtime
+      split: ${main_split}
+
+  - name: general_information
+    display_name: General information
+    hide_win_rates: true
+    metrics:
+    - name: num_instances
+      split: ${main_split}
+    - name: num_train_instances
+      split: ${main_split}
+    - name: prompt_truncated
+      split: ${main_split}
+    - name: num_prompt_tokens
+      split: ${main_split}
+    - name: num_output_tokens
+      split: ${main_split}
+
+############################################################
+run_groups:
+  - name: arabic_scenarios
+    display_name: Arabic Scenarios
+    description: Arabic Scenarios
+    category: All scenarios
+    subgroups:
+      - mmmlu
+      - arabic_mmlu
+      - alghafa
+      - exams_multilingual
+      - aratrust
+
+  - name: mmmlu
+    display_name: Multilingual MMLU (Arabic)
+    description: Multilingual MMLU (Arabic)
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: multiple-choice question answering
+      what: math, science, history, etc.
+      who: various online sources
+      when: before 2021
+      language: Arabic
+
+  - name: arabic_mmlu
+    display_name: Arabic MMLU
+    description: Arabic MMLU
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: "question answering"
+      what: "academic questions across various disciplines"
+      who: "academic exams writers and takers"
+      when: "before 2024"
+      language: Arabic
+
+  - name: alghafa
+    display_name: AlGhafa
+    description: AlGhafa
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: "multiple choice question answering"
+      what: Various
+      who: Various
+      when: "before 2023"
+      language: Arabic
+
+  - name: exams_multilingual
+    display_name: EXAMS (Arabic)
+    description: EXAMS (Arabic)
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: "multiple choice question answering"
+      what: High school examinations
+      who: High school examinations writers and test-takers
+      when: before 2020
+      language: Arabic
+
+  - name: aratrust
+    display_name: AraTrust
+    description: AraTrust
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: "question answering"
+      what: "academic questions across various disciplines"
+      who: "academic exams writers and takers"
+      when: "before 2024"
+      language: Arabic