crfm-helm 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

helm/benchmark/scenarios/infinite_bench_en_mc_scenario.py

@@ -4,6 +4,7 @@ from typing import List
 
 from datasets import load_dataset, Features, Value, Sequence, Dataset
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
     CORRECT_TAG,
     TEST_SPLIT,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -88,3 +90,22 @@ class InfiniteBenchEnMCScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="infinite_bench_en_mc",
+            display_name="∞Bench En.MC",
+            description="∞Bench En.MC is a multiple-choice question answering task that requires "
+            "locating and processing information within a novel, performing reasoning "
+            "through aggregation or filtering to derive answers. ([Zhang et al., "
+            "2024](https://arxiv.org/abs/2402.13718))",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="Novels",
+                when="Before 2024",
+                who="Novel authors",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/infinite_bench_en_sum_scenario.py

@@ -2,6 +2,7 @@ import os
 import re
 from typing import List
 from datasets import load_dataset, Features, Value, Sequence, Dataset
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -10,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
     CORRECT_TAG,
     TEST_SPLIT,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -77,3 +79,20 @@ class InfiniteBenchEnSumScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="infinite_bench_en_sum",
+            display_name="∞Bench En.Sum",
+            description="∞Bench En.Sum is a summarization task that requires generating a concise "
+            "summary of a novel. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))",
+            taxonomy=TaxonomyInfo(
+                task="multi-hop question answering",
+                what="Novels",
+                when="Before 2024",
+                who="Novel authors",
+                language="English",
+            ),
+            main_metric="rouge_l",
+            main_split="test",
+        )

helm/benchmark/scenarios/koala_scenario.py

@@ -2,8 +2,9 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, ScenarioMetadata
 
 
 class KoalaScenario(Scenario):
@@ -39,3 +40,22 @@ class KoalaScenario(Scenario):
             )
             instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="koala",
+            display_name="Koala test dataset",
+            short_display_name="Koala test dataset",
+            description="The test dataset from the [Koala "
+            "paper](https://bair.berkeley.edu/blog/2023/04/03/koala/) for evaluating "
+            "instruction-following models.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Instructions for LLMs",
+                when="Before 2023",
+                who="Web users",
+                language="English",
+            ),
+            main_metric="Helpfulness",
+            main_split="test",
+        )

helm/benchmark/scenarios/kpi_edgar_scenario.py

@@ -3,6 +3,7 @@ from typing import List, Dict
 import json
 import re
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -149,3 +151,22 @@ class KPIEDGARScenario(Scenario):
         with open(target_path, "r") as f:
             raw_dataset = json.load(f)
         return KPIEDGARScenario.sentences_to_instances(KPIEDGARScenario.get_sentences(raw_dataset))
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="kpi_edgar",
+            display_name="KPI-EDGAR Financial Documents (Named Entity Recognition)",
+            short_display_name=None,
+            description="A named entity recognition beenchmark based on the paper KPI-EDGAR - A Novel "
+            "Dataset and Accompanying Metric for Relation Extraction from Financial "
+            "Documents [(Deußer et al., 2022)](https://arxiv.org/pdf/2210.09163.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="named entity recognition",
+                what="financial reports",
+                when="before 2022",
+                who="financial experts",
+                language="English",
+            ),
+            main_metric="adjusted_macro_f1_score",
+            main_split="test",
+        )

helm/benchmark/scenarios/legal_contract_summarization_scenario.py

@@ -4,6 +4,7 @@ import json
 import re
 
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     CORRECT_TAG,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -127,3 +129,21 @@ class LegalContractSummarizationScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="legal_contract_summarization",
+            display_name="Legal Contract Summarization",
+            short_display_name=None,
+            description="Plain English Summarization of Contracts [(Manor et al., "
+            "2019)](https://aclanthology.org/W19-2201.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="summarization",
+                what="legal contracts (e.g. terms of service, license agreements)",
+                when="before 2019",
+                who="lawyers",
+                language="English",
+            ),
+            main_metric="rouge_l",
+            main_split="test",
+        )

helm/benchmark/scenarios/legal_summarization_scenario.py

@@ -5,6 +5,7 @@ from typing import List, Optional, Any
 import datasets
 from datasets import load_dataset
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -15,6 +16,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 _ALL_LANGUAGES = {
@@ -205,3 +207,51 @@ class LegalSummarizationScenario(Scenario):
                 )
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.dataset_name == "BillSum":
+            return ScenarioMetadata(
+                name="billsum_legal_summarization",
+                display_name="BillSum",
+                description="The BillSum benchmark for legal text summarization ([Kornilova & Eidelmann, "
+                "2020](https://aclanthology.org/D19-5406/)).",
+                taxonomy=TaxonomyInfo(
+                    task="summarization", what="legal text from US bills", when=None, who="lawyers", language="English"
+                ),
+                main_metric="rouge_2",
+                main_split="test",
+            )
+        elif self.dataset_name == "MultiLexSum":
+            return ScenarioMetadata(
+                name="multilexsum_legal_summarization",
+                display_name="MultiLexSum",
+                description="The MultiLexSum benchmark for legal text summarization ([Shen et al., "
+                "2022](https://arxiv.org/abs/2206.10883)).",
+                taxonomy=TaxonomyInfo(
+                    task="summarization",
+                    what="legal text from US civil rights lawsuits",
+                    when=None,
+                    who="lawyers",
+                    language="English",
+                ),
+                main_metric="rouge_2",
+                main_split="test",
+            )
+        elif self.dataset_name == "EurLexSum":
+            return ScenarioMetadata(
+                name="eurlexsum_legal_summarization",
+                display_name="EurLexSum",
+                description="The EurLexSum benchmark for legal text summarization ([Aumiller et al., "
+                "2022](https://arxiv.org/abs/2210.13448)).",
+                taxonomy=TaxonomyInfo(
+                    task="summarization",
+                    what="legal text from EU legislation",
+                    when="1960 - 2020",
+                    who="lawyers",
+                    language="English",
+                ),
+                main_metric="rouge_2",
+                main_split="test",
+            )
+        else:
+            raise Exception(f"Unknown dataset {self.dataset_name}")

helm/benchmark/scenarios/legal_support_scenario.py

@@ -2,6 +2,7 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -102,3 +104,14 @@ class LegalSupportScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="legal_support",
+            display_name="LegalSupport",
+            description="Scenario introduced in this work to measure fine-grained legal reasoning "
+            "through reverse entailment.",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/legalbench_scenario.py

@@ -5,6 +5,7 @@ import datasets
 from pathlib import Path
 from typing import List, Dict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -15,6 +16,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 PROMPT_SETTINGS_URL = "https://raw.githubusercontent.com/HazyResearch/legalbench/main/helm_prompt_settings.jsonl"
@@ -144,3 +146,20 @@ class LegalBenchScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="LegalBench",
+            description="LegalBench is a large collaboratively constructed benchmark of legal reasoning "
+            "tasks [(Guha et al, 2023)](https://arxiv.org/pdf/2308.11462.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="public legal and admininstrative documents, manually " "constructed questions",
+                when="before 2023",
+                who="lawyers",
+                language="English",
+            ),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/lex_glue_scenario.py

@@ -16,6 +16,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 ECTHR_A = "ecthr_a"
@@ -261,3 +262,13 @@ class LexGLUEScenario(Scenario):
         for subset in self.subsets:
             instances.extend(self.get_instances_for_subset(subset, output_path))
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="lex_glue",
+            display_name="LexGLUE",
+            description="A Benchmark Dataset for Legal Language Understanding in English",
+            taxonomy=None,
+            main_metric="classification_macro_f1",
+            main_split="test",
+        )

helm/benchmark/scenarios/lextreme_scenario.py

@@ -16,6 +16,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Output,
     Input,
+    ScenarioMetadata,
 )
 
 
@@ -466,3 +467,13 @@ class LEXTREMEScenario(Scenario):
         for subset in self.subsets:
             instances.extend(self.get_instances_for_subset(subset, output_path))
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="lextreme",
+            display_name="LEXTREME",
+            description="A Multilingual Legal Benchmark for Natural Language Understanding",
+            taxonomy=None,
+            main_metric="classification_macro_f1",
+            main_split="test",
+        )

helm/benchmark/scenarios/lsat_qa_scenario.py

@@ -2,6 +2,7 @@ import os
 import json
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -143,3 +145,15 @@ class LSATScenario(Scenario):
                             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="lsat_qa",
+            display_name="LSAT",
+            description="The LSAT benchmark for measuring analytical reasoning on the Law School "
+            "Admission Test (LSAT; [Zhong et al., "
+            "2021](https://arxiv.org/pdf/2104.06598.pdf)).",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/madinah_qa_scenario.py

@@ -0,0 +1,73 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MadinahQAScenario(Scenario):
+    """MadinahQA Scenario"""
+
+    name = "madinah_qa"
+    description = "Arabic language competency benchmark"
+    tags = ["language", "multiple_choice"]
+
+    OPTIONS = ["A", "B", "C", "D"]
+    HF_SPLIT_TO_HELM_SPLIT = {"dev": TRAIN_SPLIT, "test": TEST_SPLIT}
+    SUBSETS = ["Arabic Language (General)", "Arabic Language (Grammar)"]
+
+    def __init__(self, subset: str):
+        super().__init__()
+        subset = subset.replace("_", " ")
+        if subset not in self.SUBSETS:
+            raise Exception(f"Unknown subset: {subset}")
+        self.subset = subset
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        instances: List[Instance] = []
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "MBZUAI/MadinahQA",
+            self.subset,
+            revision="62e7c86ac5c07245a5a952722691d77ddb41f695",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            for row_index, row in enumerate(dataset):
+                input = Input(text=row["Question"])
+                references: List[Reference] = []
+                correct_option_index = ord(row["Answer Key"]) - ord("A") + 1
+                for option_index in range(1, 6):
+                    column_name = f"Option {option_index}"
+                    if not row[column_name]:
+                        continue
+                    references.append(
+                        Reference(
+                            output=Output(text=row[column_name]),
+                            tags=[CORRECT_TAG] if option_index == correct_option_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=f"id{row_index}",
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/math_scenario.py

@@ -4,6 +4,7 @@ import typing
 from typing import Dict, List, Optional
 from datasets import load_dataset, DatasetDict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -450,3 +452,34 @@ class MATHScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        taxonomy = TaxonomyInfo(
+            task="numeric answer question answering",
+            what="math competitions (AMC, AIME, etc.)",
+            when="before 2021",
+            who="problem setters",
+            language="synthetic",
+        )
+        if self.use_chain_of_thought:
+            return ScenarioMetadata(
+                name="math_chain_of_thought",
+                display_name="MATH",
+                description="The MATH benchmark for measuring mathematical problem solving on competition "
+                "math problems with chain-of-thought style reasoning [(Hendrycks et al., "
+                "2021)](https://arxiv.org/pdf/2103.03874.pdf).",
+                taxonomy=taxonomy,
+                main_metric="math_equiv_chain_of_thought",
+                main_split="test",
+            )
+        else:
+            return ScenarioMetadata(
+                name="math_regular",
+                display_name="MATH",
+                description="The MATH benchmark for measuring mathematical problem solving on competition "
+                "math problems [(Hendrycks et al., "
+                "2021)](https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/hash/be83ab3ecd0db773eb2dc1b0a17836a1-Abstract-round2.html).",
+                taxonomy=taxonomy,
+                main_metric="math_equiv",
+                main_split="test",
+            )

helm/benchmark/scenarios/mbzuai_human_translated_arabic_mmlu.py

@@ -0,0 +1,68 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MBZUAIHumanTranslatedArabicMMLUScenario(Scenario):
+    """MBZUAI Human-Translated Arabic MMLU
+
+    A translation from MBZUAI by human translators of the Massive Multitask Language Understanding benchmark from this paper:
+
+    - https://arxiv.org/pdf/2009.03300.pdf
+    """  # noqa: E501
+
+    name = "mbzuai_human_translated_arabic_mmlu"
+    description = (
+        "A translation from MBZUAI by human translators of the Massive Multitask Language Understanding benchmark"
+    )
+    tags = ["knowledge", "multiple_choice"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        self.subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "MBZUAI/human_translated_arabic_mmlu",
+            self.subject,
+            revision="5ed7830fd678cfa6f2d7f0a1a13a4e1a1fa422ac",
+            cache_dir=cache_dir,
+            split="test",
+        )
+        assert isinstance(dataset, datasets.Dataset)
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row_index, row in enumerate(dataset):
+            input = Input(text=row["question"])
+            references: List[Reference] = []
+            for choice_index, choice in enumerate(row["choices"]):
+                references.append(
+                    Reference(
+                        output=Output(text=choice),
+                        tags=[CORRECT_TAG] if choice_index == row["answer"] else [],
+                    )
+                )
+            instance = Instance(
+                id=f"id-{self.subject}-{row_index}",
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/med_dialog_scenario.py

@@ -2,8 +2,18 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Reference, ALL_SPLITS, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    ALL_SPLITS,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class MedDialogScenario(Scenario):
@@ -133,3 +143,24 @@ class MedDialogScenario(Scenario):
                         )
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="med_dialog",
+            display_name="MedDialog",
+            short_display_name="MedDialog",
+            description="MedDialog is a benchmark of real-world doctor-patient conversations focused on "
+            "health-related concerns and advice. Each dialogue is paired with a "
+            "one-sentence summary that reflects the core patient question or exchange. The "
+            "benchmark evaluates a model's ability to condense medical dialogue into "
+            "concise, informative summaries.",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Generate summaries of doctor-patient conversations",
+                when="Any",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="med_dialog_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/med_mcqa_scenario.py

@@ -2,6 +2,7 @@ import json
 import os
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     VALID_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -109,3 +111,15 @@ class MedMCQAScenario(Scenario):
                     instances.append(instance)
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="med_mcqa",
+            display_name="MedMCQA",
+            description='MedMCQA is a "multiple-choice question answering (MCQA) dataset designed to '
+            "address real-world medical entrance exam questions ([Flores et al. "
+            "2020](https://arxiv.org/abs/2203.14371)).",
+            taxonomy=TaxonomyInfo(task="question answering", what="n/a", when="n/a", who="n/a", language="English"),
+            main_metric="exact_match",
+            main_split="valid",
+        )