crfm-helm 0.5.7__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/scenarios/aci_bench_scenario.py

@@ -1,6 +1,7 @@
 import json
 import os
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -10,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_file_downloaded
 
@@ -124,3 +126,24 @@ class ACIBenchScenario(Scenario):
             instances.extend(self.process_json(test_json, TEST_SPLIT))
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="aci_bench",
+            display_name="ACI-Bench",
+            description="ACI-Bench is a benchmark of real-world patient-doctor conversations paired "
+            "with structured clinical notes. The benchmark evaluates a model's ability to "
+            "understand spoken medical dialogue and convert it into formal clinical "
+            "documentation, covering sections such as history of present illness, physical "
+            "exam findings, results, and assessment and plan [(Yim et al., "
+            "2024)](https://www.nature.com/articles/s41597-023-02487-3).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Extract and structure information from patient-doctor " "conversations",
+                when="Any",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="aci_bench_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/air_bench_scenario.py

@@ -2,6 +2,7 @@ import datasets
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -9,6 +10,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -53,3 +55,22 @@ class AIRBench2024Scenario(Scenario):
             instance = Instance(input=input, references=references, split=TEST_SPLIT)
             instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="air_bench_2024",
+            display_name="AIRBench 2024",
+            description="AIRBench 2024 is a AI safety benchmark that aligns with emerging government "
+            "regulations and company policies. It consists of diverse, malicious prompts "
+            "spanning categories of the regulation-based safety categories in the AIR 2024 "
+            "safety taxonomy.\n",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction-following text generation",
+                what="malicious prompts",
+                when="2024",
+                who="dataset authors and language models",
+                language="English",
+            ),
+            main_metric="air_score",
+            main_split="test",
+        )

helm/benchmark/scenarios/alrage_scenario.py

@@ -0,0 +1,54 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class ALRAGEScenario(Scenario):
+    """ALRAGE"""  # noqa: E501
+
+    name = "alrage"
+    description = "ALRAGE"
+    tags = ["open-book question answering"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset: datasets.Dataset = datasets.load_dataset(
+            "OALL/ALRAGE",
+            revision="4827b2ed2436aea578e84d9bd4150b66ab8bbe0e",
+            split="train",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=f"السؤال:\n{row['question']}\n\nالسياقات المقترحة:\n{row['candidates']}\n")
+            references: List[Reference] = []
+            references = [
+                Reference(
+                    output=Output(text=row["gold_answer"]),
+                    tags=[CORRECT_TAG],
+                )
+            ]
+            instance = Instance(
+                id=row["id"],
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/anthropic_hh_rlhf_scenario.py

@@ -2,7 +2,8 @@ import re
 from typing import List, Any, Dict
 from datasets import load_dataset
 
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TRAIN_SPLIT, TEST_SPLIT
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TRAIN_SPLIT, TEST_SPLIT, ScenarioMetadata
 
 
 class AnthropicHHRLHFScenario(Scenario):
@@ -88,3 +89,24 @@ class AnthropicHHRLHFScenario(Scenario):
                 )
                 instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="anthropic_hh_rlhf",
+            display_name="Anthropic RLHF dataset",
+            short_display_name="Anthropic RLHF dataset",
+            description="The dialogue datasets released by Anthropic to facilitate research in model "
+            "helpfulness and harmlessness ([Bai et al., "
+            "2022](https://arxiv.org/pdf/2204.05862.pdf); [Ganguli et al., "
+            "2022](https://arxiv.org/pdf/2209.07858.pdf)). We only use the first utterance "
+            "of each dialogue.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Human-LM dialogues and preference labels",
+                when="2022",
+                who="Workers from MTurk and Upwork, language models from Anthropic",
+                language="English",
+            ),
+            main_metric="Helpfulness",
+            main_split="test",
+        )

helm/benchmark/scenarios/arabic_exams_scenario.py

@@ -0,0 +1,114 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.hierarchical_logger import hwarn
+
+
+class ArabicEXAMSScenario(Scenario):
+    """The Arabic subset of the EXAMS High School Examinations Dataset for Multilingual Question Answering
+
+    We use the Open Arabic LLM Leaderboard (OALL) version mirror of the Arabic subset of EXAMS, which is in-turn based
+    on the AceGPT version.
+
+    See: https://www.tii.ae/news/introducing-open-arabic-llm-leaderboard-empowering-arabic-language-modeling-community
+
+    References:
+
+    ```
+    @misc{huang2024acegptlocalizinglargelanguage,
+        title={AceGPT, Localizing Large Language Models in Arabic},
+        author={Huang Huang and Fei Yu and Jianqing Zhu and Xuening Sun and Hao Cheng and Dingjie Song and Zhihong Chen and Abdulmohsen Alharthi and Bang An and Juncai He and Ziche Liu and Zhiyi Zhang and Junying Chen and Jianquan Li and Benyou Wang and Lian Zhang and Ruoyu Sun and Xiang Wan and Haizhou Li and Jinchao Xu},
+        year={2024},
+        eprint={2309.12053},
+        archivePrefix={arXiv},
+        primaryClass={cs.CL},
+        url={https://arxiv.org/abs/2309.12053},
+    }```
+
+    ```
+    @inproceedings{hardalov-etal-2020-exams,
+        title = "{EXAMS}: A Multi-subject High School Examinations Dataset for Cross-lingual and Multilingual Question Answering",
+        author = "Hardalov, Momchil  and
+        Mihaylov, Todor  and
+        Zlatkova, Dimitrina  and
+        Dinkov, Yoan  and
+        Koychev, Ivan  and
+        Nakov, Preslav",
+        editor = "Webber, Bonnie  and
+        Cohn, Trevor  and
+        He, Yulan  and
+        Liu, Yang",
+        booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
+        month = nov,
+        year = "2020",
+        address = "Online",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2020.emnlp-main.438/",
+        doi = "10.18653/v1/2020.emnlp-main.438",
+        pages = "5427--5444",
+        abstract = "We propose EXAMS {--} a new benchmark dataset for cross-lingual and multilingual question answering for high school examinations. We collected more than 24,000 high-quality high school exam questions in 16 languages, covering 8 language families and 24 school subjects from Natural Sciences and Social Sciences, among others.EXAMS offers unique fine-grained evaluation framework across multiple languages and subjects, which allows precise analysis and comparison of the proposed models. We perform various experiments with existing top-performing multilingual pre-trained models and show that EXAMS offers multiple challenges that require multilingual knowledge and reasoning in multiple domains. We hope that EXAMS will enable researchers to explore challenging reasoning and knowledge transfer methods and pre-trained models for school question answering in various languages which was not possible by now. The data, code, pre-trained models, and evaluation are available at http://github.com/mhardalov/exams-qa."
+    }```
+    """  # noqa: E501
+
+    name = "arabic_exams"
+    description = "EXAMS is a benchmark dataset for multilingual and cross-lingual question answering from high school examinations. "  # noqa: E501
+    tags = ["knowledge", "multiple_choice"]
+
+    CHOICES = ["A", "B", "C", "D"]
+    # Remap validation split to train split
+    HF_SPLIT_TO_HELM_SPLIT = {"validation": TRAIN_SPLIT, "test": TEST_SPLIT}
+
+    def __init__(self, subject: str):
+        super().__init__()
+        self.subject: str = subject.replace("_", " ")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits = datasets.load_dataset(
+            "OALL/Arabic_EXAMS",
+            revision="bc7a29346dbcaa16a8cd883b1f3e681ab2b7ff2a",
+            cache_dir=cache_dir,
+        )
+
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            for row in dataset:
+                subject = row["id"].split("-")[0]
+                if self.subject != "all" and self.subject != subject:
+                    continue
+                input = Input(text=row["question"])
+                references: List[Reference] = []
+                if row["answer"] not in self.CHOICES:
+                    hwarn(f"Invalid value in answer column in row: {row}")
+                    continue
+                correct_choice = row["answer"]
+                for choice in self.CHOICES:
+                    references.append(
+                        Reference(
+                            output=Output(text=row[choice]),
+                            tags=[CORRECT_TAG] if choice == correct_choice else [],
+                        )
+                    )
+                instance = Instance(
+                    id=row["id"],
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/arabic_mmlu_scenario.py

@@ -19,8 +19,6 @@ from helm.benchmark.scenarios.scenario import (
 class ArabicMMLUScenario(Scenario):
     """ArabicMMLU
 
-    EXPERIMENTAL: This scenario may have future reverse incompatible changes.
-
     ArabicMMLU is the first multi-task language understanding benchmark
     for Arabic language, sourced from school exams across diverse educational
     levels in different countries spanning North Africa, the Levant, and the
@@ -39,12 +37,16 @@ class ArabicMMLUScenario(Scenario):
     OPTIONS = ["A", "B", "C", "D"]
     HF_SPLIT_TO_HELM_SPLIT = {"dev": TRAIN_SPLIT, "test": TEST_SPLIT}
 
+    def __init__(self, subset: str):
+        super().__init__()
+        self.subset = subset.replace("_", " ")
+
     def get_instances(self, output_path: str) -> List[Instance]:
         cache_dir = os.path.join(output_path, "data")
         ensure_directory_exists(cache_dir)
         dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
             "MBZUAI/ArabicMMLU",
-            "All",
+            self.subset,
             revision="7aa530e2893ac420352b3f5c1a1310c010e9758b",
             cache_dir=cache_dir,
         )
@@ -63,7 +65,9 @@ class ArabicMMLUScenario(Scenario):
                         continue
                     references.append(
                         Reference(
-                            output=Output(text=row[column_name]),
+                            # Need to convert column to string because the references are floats
+                            # for the subject "Math (Primary School)"
+                            output=Output(text=str(row[column_name])),
                             tags=[CORRECT_TAG] if option_index == correct_option_index else [],
                         )
                     )

helm/benchmark/scenarios/aratrust_scenario.py

@@ -47,8 +47,25 @@ class AraTrustScenario(Scenario):
     description = "aratrust"
     tags = ["trustworthiness"]
 
+    CATEGORIES = [
+        "Ethics",
+        "Illegal",
+        "Mental Health",
+        "Offensive",
+        "Physical Health",
+        "Privacy",
+        "Trustfulness",
+        "Unfairness",
+    ]
     OPTION_KEYS = ["A", "B", "C"]
 
+    def __init__(self, category: str):
+        super().__init__()
+        category = category.replace("_", " ")
+        if category not in self.CATEGORIES and category != "all":
+            raise Exception(f"Unknown category {category}")
+        self.category = category
+
     def get_instances(self, output_path: str) -> List[Instance]:
         cache_dir = os.path.join(output_path, "data")
         ensure_directory_exists(cache_dir)
@@ -60,6 +77,8 @@ class AraTrustScenario(Scenario):
         )
         instances: List[Instance] = []
         for row_index, row in enumerate(dataset):
+            if self.category != "all" and self.category != row["Category"]:
+                continue
             question_text = row["Question"]
             option_texts = [row[option_key] for option_key in self.OPTION_KEYS if row[option_key]]
             joined_option_texts = "\n".join(option_texts)

helm/benchmark/scenarios/babi_qa_scenario.py

@@ -1,6 +1,7 @@
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -139,3 +141,16 @@ class BabiQAScenario(Scenario):
                             story.append(fact)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="babi_qa",
+            display_name="bAbI",
+            description="The bAbI benchmark for measuring understanding and reasoning [(Weston et al., "
+            "2015)](https://arxiv.org/pdf/1502.05698.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="question answering", what="reasoning", when="2015", who="synthetic", language="English"
+            ),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/bbq_scenario.py

@@ -3,6 +3,7 @@ import os
 import random
 from typing import List, Dict, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     DEFAULT_TEST_SIZE,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 AMBIGUOUS_TAG = "ambiguous"
@@ -237,3 +239,16 @@ class BBQScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="bbq",
+            display_name="BBQ (Bias Benchmark for Question Answering)",
+            short_display_name="BBQ",
+            description="The Bias Benchmark for Question Answering (BBQ) for measuring social bias in "
+            "question answering in ambiguous and unambigous context [(Parrish et al., "
+            "2022)](https://aclanthology.org/2022.findings-acl.165/).",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )