crfm-helm 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

helm/benchmark/scenarios/thai_exam_scenario.py

@@ -2,6 +2,7 @@ import os
 from typing import Dict, List
 import json
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.scenario import (
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -142,3 +144,96 @@ class ThaiExamScenario(Scenario):
             instances.extend(self.process_jsonl(jsonl_path, splits[split]))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.exam == "onet":
+            return ScenarioMetadata(
+                name="thai_exam_onet",
+                display_name="ONET",
+                description="The Ordinary National Educational Test (ONET) is an examination for students "
+                "in Thailand. We select the grade-12 ONET exam, which comprises 5 subjects and "
+                "each question has 5 choices. These subjects are Thai, English, Mathematics, "
+                "Social Studies, and Science. Amounting to a total of 170 questions and "
+                "options.\n",
+                taxonomy=TaxonomyInfo(
+                    task="question answering",
+                    what="high school / medical school academic knowledge",
+                    when="?",
+                    who="n/a",
+                    language="Thai and English",
+                ),
+                main_metric="exact_match",
+                main_split="test",
+            )
+        elif self.exam == "ic":
+            return ScenarioMetadata(
+                name="thai_exam_ic",
+                display_name="IC",
+                description="The Investment Consultant (IC) examination, a licensing test for investment "
+                "professionals in Thailand. Developed by the Stock Exchange of Thailand (SET), "
+                "features 4 choices per question. We extracted questions for levels 1, 2, and 3 "
+                "resulting in a total of 95 questions and options.\n",
+                taxonomy=TaxonomyInfo(
+                    task="question answering",
+                    what="licensing for investment professionals",
+                    when="?",
+                    who="n/a",
+                    language="Thai",
+                ),
+                main_metric="exact_match",
+                main_split="test",
+            )
+        elif self.exam == "tgat":
+            return ScenarioMetadata(
+                name="thai_exam_tgat",
+                display_name="TGAT",
+                description="The Thai General Aptitude Test (TGAT), a national high school examination in "
+                "Thailand. Focuses on critical and logical thinking skills. We collected a "
+                "total of 90 questions and answers. The TGAT consists of four choices per "
+                "question.\n",
+                taxonomy=TaxonomyInfo(
+                    task="question answering",
+                    what="high school level questions on reasoning",
+                    when="?",
+                    who="n/a",
+                    language="English",
+                ),
+                main_metric="exact_match",
+                main_split="test",
+            )
+        elif self.exam == "tpat1":
+            return ScenarioMetadata(
+                name="thai_exam_tpat1",
+                display_name="TPAT-1",
+                description="TBD",
+                taxonomy=TaxonomyInfo(
+                    task="question answering",
+                    what="high school / medical school academic knowledge",
+                    when="?",
+                    who="n/a",
+                    language="Thai",
+                ),
+                main_metric="exact_match",
+                main_split="test",
+            )
+        elif self.exam == "a_level":
+            return ScenarioMetadata(
+                name="thai_exam_a_level",
+                display_name="A-Level",
+                description="An academic knowledge assessment examination (Applied Knowledge Level) that "
+                "covers general foundational subjects taught in schools. The content assessed "
+                "in this examination aligns with the curriculum guidelines and emphasizes the "
+                "practical application of knowledge in daily life. We collected a total of 175 "
+                "questions and answers.\n",
+                taxonomy=TaxonomyInfo(
+                    task="question answering",
+                    what="high school academic knowledge",
+                    when="?",
+                    who="n/a",
+                    language="Thai and English",
+                ),
+                main_metric="exact_match",
+                main_split="test",
+            )
+        else:
+            raise ValueError(f"Unknown exam: {self.exam}")

helm/benchmark/scenarios/the_pile_scenario.py

@@ -5,9 +5,10 @@ import sys
 import requests
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog, htrack, htrack_block
-from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input, ScenarioMetadata
 
 
 class ThePileScenario(Scenario):
@@ -146,3 +147,14 @@ class ThePileScenario(Scenario):
             instances = [instances[i] for i in indices]
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="the_pile",
+            display_name="The Pile",
+            description="The Pile corpus for measuring lanugage model performance across various "
+            "domains [(Gao et al., 2020)](https://arxiv.org/pdf/2101.00027.pdf).",
+            taxonomy=TaxonomyInfo(task="language modeling", what="?", when="?", who="?", language="English, code"),
+            main_metric="bits_per_byte",
+            main_split="test",
+        )

helm/benchmark/scenarios/truthful_qa_scenario.py

@@ -2,6 +2,7 @@ import csv
 import os
 from typing import List, Dict, Any
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -154,3 +156,15 @@ class TruthfulQAScenario(Scenario):
         valid_instances: List[Instance] = get_split_instances(VALID_SPLIT, data[split_k:])
 
         return train_instances + valid_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="truthful_qa",
+            display_name="TruthfulQA",
+            description="The TruthfulQA benchmarking for measuring model truthfulness and commonsense "
+            "knowledge in question answering [(Lin et al., "
+            "2022)](https://aclanthology.org/2022.acl-long.229/).",
+            taxonomy=TaxonomyInfo(task="question answering", what="?", when="?", who="?", language="English"),
+            main_metric="exact_match",
+            main_split="valid",
+        )

helm/benchmark/scenarios/twitter_aae_scenario.py

@@ -2,9 +2,10 @@ import csv
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input, ScenarioMetadata
 
 CODALAB_URI_TEMPLATE: str = (
     "https://worksheets.codalab.org/rest/bundles/0x31485f8c37ad481fb9f4e9bf7ccff6e5/contents/blob/"
@@ -56,3 +57,21 @@ class TwitterAAEScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="twitter_aae",
+            display_name="TwitterAAE",
+            description="The TwitterAAE corpus of [Blodgett et al. "
+            "(2016)](https://aclanthology.org/D16-1120/) for measuring language model "
+            "performance in tweets as a function of speaker dialect.",
+            taxonomy=TaxonomyInfo(
+                task="language modeling",
+                what="?",
+                when="?",
+                who="?",
+                language="English (AAE-aligned and White-aligned)",
+            ),
+            main_metric="bits_per_byte",
+            main_split="test",
+        )

helm/benchmark/scenarios/vicuna_scenario.py

@@ -2,8 +2,9 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, ScenarioMetadata
 
 
 class VicunaScenario(Scenario):
@@ -47,3 +48,22 @@ class VicunaScenario(Scenario):
                 )
                 instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="vicuna",
+            display_name="Vicuna",
+            short_display_name="Vicuna",
+            description="The set of prompts used by the "
+            "[Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) team to evaluate "
+            "instruction-following models.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Instructions for LLMs",
+                when="Before 2023",
+                who="Unknown",
+                language="English",
+            ),
+            main_metric="Helpfulness",
+            main_split="test",
+        )

helm/benchmark/scenarios/wikifact_scenario.py

@@ -2,6 +2,7 @@ import os
 from typing import List, Dict
 import json
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded, flatten_list
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.scenario import (
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 PID_TO_NAME = {
@@ -183,3 +185,21 @@ class WIKIFactScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="wikifact",
+            display_name="WikiFact",
+            description="Scenario introduced in this work, inspired by [Petroni et al. "
+            "(2019)](https://aclanthology.org/D19-1250/), to more extensively test factual "
+            "knowledge.",
+            taxonomy=TaxonomyInfo(
+                task="knowledge base completion",
+                what="entity-relation-entity triples in natural language form",
+                when="?",
+                who="automatically generated from templates",
+                language="structured English",
+            ),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/wildbench_scenario.py

@@ -2,11 +2,13 @@ import datasets
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     TEST_SPLIT,
     Input,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -81,3 +83,19 @@ class WildBenchScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="WildBench",
+            description=self.description,
+            main_metric="wildbench_score_rescaled",
+            main_split="test",
+            taxonomy=TaxonomyInfo(
+                task="instruction following",
+                what="GPT-judged instruction following with instructions collected from real-user conversations",
+                who="real-world users",
+                when="2024",
+                language="English",
+            ),
+        )

helm/benchmark/scenarios/wmt_14_scenario.py

@@ -1,5 +1,6 @@
 from typing import List, Any
 from datasets import load_dataset
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.hierarchical_logger import htrack_block
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -106,3 +108,20 @@ class WMT14Scenario(Scenario):
                     )
                 )
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="wmt_14",
+            display_name="WMT 2014",
+            description="WMT 2014 is a collection of machine translation datasets "
+            "[(website)](https://www.statmt.org/wmt14/index.html).",
+            taxonomy=TaxonomyInfo(
+                task="machine translation",
+                what="multilingual sentences",
+                when="before 2014",
+                who="Europarl, news, Common Crawl, etc.",
+                language="English, French, Czech, etc.",
+            ),
+            main_metric="bleu_4",
+            main_split="test",
+        )

helm/benchmark/static/schema_arabic.yaml

@@ -92,6 +92,12 @@ metrics:
     short_display_name: PEM
     description: Fraction of instances that the predicted output matches the prefix of a correct reference up to light processing.
     lower_is_better: false
+  - name: alrage_score
+    # TODO: should call this prefix_quasi_exact_match
+    display_name: ALRAGE Score
+    short_display_name: Score
+    description: Score of the output judged by GPT-4o.
+    lower_is_better: false
 
 ############################################################
 perturbations: []
@@ -134,17 +140,20 @@ run_groups:
   - name: arabic_scenarios
     display_name: Arabic Scenarios
     description: Arabic Scenarios
-    category: All scenarios
+    category: Scenarios
     subgroups:
-      - mmmlu
-      - arabic_mmlu
       - alghafa
-      - exams_multilingual
+      - arabic_mmlu
+      - arabic_exams
+      - madinah_qa
       - aratrust
+      - alrage
+      - mbzuai_human_translated_arabic_mmlu
 
-  - name: mmmlu
-    display_name: Multilingual MMLU (Arabic)
-    description: Multilingual MMLU (Arabic)
+  - name: mbzuai_human_translated_arabic_mmlu
+    display_name: MBZUAI Human-Translated Arabic MMLU
+    short_display_name: Translated MMLU
+    description: A translation from MBZUAI by human translators of the Massive Multitask Language Understanding benchmark.
     metric_groups:
       - accuracy
       - efficiency
@@ -160,8 +169,8 @@ run_groups:
       language: Arabic
 
   - name: arabic_mmlu
-    display_name: Arabic MMLU
-    description: Arabic MMLU
+    display_name: ArabicMMLU
+    description: ArabicMMLU
     metric_groups:
       - accuracy
       - efficiency
@@ -193,9 +202,9 @@ run_groups:
       when: "before 2023"
       language: Arabic
 
-  - name: exams_multilingual
-    display_name: EXAMS (Arabic)
-    description: EXAMS (Arabic)
+  - name: arabic_exams
+    display_name: Arabic EXAMS
+    description: Arabic EXAMS
     metric_groups:
       - accuracy
       - efficiency
@@ -226,3 +235,37 @@ run_groups:
       who: "academic exams writers and takers"
       when: "before 2024"
       language: Arabic
+
+  - name: alrage
+    display_name: ALRAGE
+    description: ALRAGE
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: alrage_score
+      main_split: test
+    taxonomy:
+      task: "openbook (RAG) open-ended question answering"
+      what: "?"
+      who: "?"
+      when: "?"
+      language: Arabic
+
+  - name: madinah_qa
+    display_name: MadinahQA
+    description: Arabic language competency benchmark
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: "question answering"
+      what: "academic questions about Arabic language"
+      who: "academic exams writers and takers"
+      when: "before 2024"
+      language: Arabic

helm/benchmark/static/schema_long_context.yaml

@@ -191,31 +191,12 @@ run_groups:
     description: Scenarios for evaluating long context capabilities
     category: All scenarios
     subgroups:
-      - ruler_hotpotqa
       - ruler_squad
-      - infinite_bench_en_sum
-      - infinite_bench_en_qa
+      - ruler_hotpotqa
       - infinite_bench_en_mc
+      - infinite_bench_en_sum
       - openai_mrcr
 
-  - name: ruler_hotpotqa
-    display_name: RULER HotPotQA
-    description: RULER HotPotQA is an augmented version of HotPotQA ([Yang et al., 2018](https://arxiv.org/abs/1809.09600)) introduced by [Hsieh et al., 2024](https://arxiv.org/abs/2404.06654) to simulate a multi-hop question answering as a long-context scenario.
-    metric_groups:
-      - accuracy
-      - general_information
-      - annotation_metrics
-    environment:
-      main_name: ruler_string_match_part
-      main_split: valid
-    taxonomy:
-      task: question answering with retrieval-augmented generation
-      what: Wikipedia articles
-      who: Wikipedia authors
-      when: Before 2018
-      language: English
-
-
   - name: ruler_squad
     display_name: RULER SQuAD
     description: RULER SQuAD is an augmented version of SQuAD ([Rajpurkar et al., 2018](https://arxiv.org/abs/1806.03822)) introduced by [Hsieh et al., 2024](https://arxiv.org/abs/2404.06654) to simulate a single-hop question answering as a long-context scenario.
@@ -233,21 +214,21 @@ run_groups:
       when: Before 2018
       language: English
 
-  - name: infinite_bench_en_qa
-    display_name: ∞Bench En.QA
-    description: ∞Bench En.QA is a open-ended question answering task that requires locating and processing information within a novel, performing reasoning through aggregation or filtering to derive answers. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))
+  - name: ruler_hotpotqa
+    display_name: RULER HotPotQA
+    description: RULER HotPotQA is an augmented version of HotPotQA ([Yang et al., 2018](https://arxiv.org/abs/1809.09600)) introduced by [Hsieh et al., 2024](https://arxiv.org/abs/2404.06654) to simulate a multi-hop question answering as a long-context scenario.
     metric_groups:
       - accuracy
       - general_information
       - annotation_metrics
     environment:
-      main_name: f1_score
-      main_split: test
+      main_name: ruler_string_match_part
+      main_split: valid
     taxonomy:
-      task: question answering
-      what: Novels
-      who: Novel authors
-      when: Before 2024
+      task: question answering with retrieval-augmented generation
+      what: Wikipedia articles
+      who: Wikipedia authors
+      when: Before 2018
       language: English
 
   - name: infinite_bench_en_mc

helm/benchmark/static/schema_medhelm.yaml

@@ -484,6 +484,8 @@ run_groups:
       - ehrshot
       - head_qa
       - medbullets
+      - med_qa
+      - med_mcqa
       - medalign
       - shc_ptbm_med
       - shc_sei_med
@@ -657,6 +659,40 @@ run_groups:
       when: Any
       language: English
 
+  - name: med_qa
+    display_name: MedQA
+    description: MedQA is an open domain question answering dataset composed of questions from professional medical board exams ([Jin et al. 2020](https://arxiv.org/pdf/2009.13081.pdf)).
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: test
+    taxonomy:
+      task: question answering
+      what: n/a
+      who: n/a
+      when: n/a
+      language: English
+
+  - name: med_mcqa
+    display_name: MedMCQA
+    description: MedMCQA is a "multiple-choice question answering (MCQA) dataset designed to address real-world medical entrance exam questions ([Flores et al. 2020](https://arxiv.org/abs/2203.14371)).
+    metric_groups:
+      - accuracy
+      - efficiency
+      - general_information
+    environment:
+      main_name: exact_match
+      main_split: valid
+    taxonomy:
+      task: question answering
+      what: n/a
+      who: n/a
+      when: n/a
+      language: English
+
   - name: medalign
     display_name: MedAlign
     short_display_name: MedAlign