crfm-helm 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/scenarios/decodingtrust_toxicity_prompts_scenario.py

@@ -3,8 +3,9 @@ import os
 import random
 from typing import List, Dict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input, ScenarioMetadata
 
 
 DATA_REPO_HASH = "38972f6ccbf376a8d0660babafb4d2b3b9cca3f4"
@@ -76,3 +77,14 @@ class DecodingTrustToxicityPromptsScenario(Scenario):
         random.shuffle(instances)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="decodingtrust_toxicity_prompts",
+            display_name="DecodingTrust - Toxicity",
+            short_display_name="Toxicity",
+            description="Evaluation of the privacy understanding and privacy preserving properties of " "LLMs",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="unknown",
+            main_split="test",
+        )

helm/benchmark/scenarios/dischargeme_scenario.py

@@ -1,4 +1,5 @@
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
@@ -8,6 +9,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
     Output,
+    ScenarioMetadata,
 )
 import pandas as pd
 
@@ -170,3 +172,25 @@ class DischargeMeScenario(Scenario):
             lines = file.readlines()
         lines = [line.strip() for line in lines]
         return lines
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="dischargeme",
+            display_name="DischargeMe",
+            short_display_name="DischargeMe",
+            description="DischargeMe is a benchmark designed to evaluate clinical text generation. It "
+            "pairs discharge summaries and radiology reports from MIMIC-IV with generation "
+            "tasks such as writing discharge instructions or summarizing the brief hospital "
+            "course. The benchmark assesses a model's ability to generate patient-facing "
+            "documentation that is complete, empathetic, and clinically accurate [(Xu, "
+            "2024)](https://physionet.org/content/discharge-me/1.3/).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Generate discharge instructions from hospital notes",
+                when="Upon hospital discharge",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="dischargeme_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/disinformation_scenario.py

@@ -2,6 +2,7 @@ import json
 import os
 from typing import List, Dict, Optional
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 REITERATION_DATA_URL = "https://drive.google.com/uc?export=download&id=1uVJbsgPCHFAvH43I6SVvU3Ayo8dh-y_N"
@@ -175,3 +177,23 @@ class DisinformationScenario(Scenario):
             instances = self.create_wedging_instances(data)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.capability == "reiteration":
+            name = "disinformation_reiteration"
+            display_name = "Disinformation (reiteration)"
+        elif self.capability == "wedging":
+            name = "disinformation_wedging"
+            display_name = "Disinformation (wedging)"
+        else:
+            raise Exception(f"Unknown capability {self.capability}")
+        return ScenarioMetadata(
+            name=name,
+            display_name=display_name,
+            description="Scenario from [Buchanan et al. "
+            "(2021)](https://cset.georgetown.edu/publication/truth-lies-and-automation/) "
+            "that tests the ability to generate divisive and wedging content.",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="unknown",
+            main_split="valid",
+        )

helm/benchmark/scenarios/dyck_language_scenario.py

@@ -2,6 +2,7 @@ import numpy as np
 import random
 from typing import List, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -234,3 +236,16 @@ class DyckLanguageScenario(Scenario):
             not_allowed=train_inputs,
         )
         return train_instances + test_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="dyck_language",
+            display_name="Dyck",
+            description="Scenario testing hierarchical reasoning through the Dyck formal languages "
+            "[(Suzgun et al., 2019)](https://aclanthology.org/W19-3905/).",
+            taxonomy=TaxonomyInfo(
+                task="next-word prediction", what="Dyck formal language", when="n/a", who="n/a", language="synthetic"
+            ),
+            main_metric="exact_match_indicator",
+            main_split="test",
+        )

helm/benchmark/scenarios/ehrshot_scenario.py

@@ -7,6 +7,7 @@ from functools import partial
 from tqdm import tqdm
 from typing import Any, Dict, List, Optional, Mapping
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import check_file_exists, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
@@ -16,6 +17,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
     Output,
+    ScenarioMetadata,
 )
 
 ##################################
@@ -1517,3 +1519,23 @@ class EHRSHOTScenario(Scenario):
             )
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="ehrshot",
+            display_name="EHRSHOT",
+            description="EHRSHOT is a benchmark designed to evaluate a model's ability to predict "
+            "future clinical events using structured EHR code sequences. Each instance "
+            "contains a patient's historical EHR data and a forward-looking clinical "
+            "question about whether a particular diagnosis, lab result, or hospital event "
+            "will occur [(Wornow et al., 2023)](https://arxiv.org/abs/2307.02028).",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Predict whether a medical event will occur in the future based " "on EHR codes",
+                when="Future prediction",
+                who="Clinician, Insurer",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/enem_challenge_scenario.py

@@ -2,6 +2,7 @@ from typing import List, Any
 from pathlib import Path
 from datasets import load_dataset
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -10,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -56,3 +58,20 @@ class ENEMChallengeScenario(Scenario):
             )
             instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="enem_challenge",
+            display_name="ENEM Challenge",
+            short_display_name=None,
+            description="ENEM Challenge",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="general academic subjects",
+                when="between 2009 and 2023",
+                who="brazilian ministry of education",
+                language="Portuguese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/entity_data_imputation_scenario.py

@@ -3,6 +3,7 @@ import pandas as pd
 from pathlib import Path
 from typing import List, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.hierarchical_logger import hlog
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
@@ -15,6 +16,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -160,3 +162,15 @@ class EntityDataImputationScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="entity_data_imputation",
+            display_name="Data imputation",
+            description="Scenario from [Mei et al. "
+            "(2021)](https://ieeexplore.ieee.org/document/9458712/) that tests the ability "
+            "to impute missing entities in a data table.",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/entity_matching_scenario.py

@@ -2,6 +2,7 @@ import pandas as pd
 from pathlib import Path
 from typing import Dict, List, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.hierarchical_logger import hlog
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.benchmark.scenarios.entity_matching_scenario_fixed_random_state import set_fixed_random_state_for_dataset
 
@@ -155,3 +157,15 @@ class EntityMatchingScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="entity_matching",
+            display_name="Entity matching",
+            description="Scenario from Magellan [(Konda et al., "
+            "2016)](https://dl.acm.org/doi/10.14778/3007263.3007314) that tests the ability "
+            "to determine if two entities match.",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/exams_multilingual_scenario.py

@@ -0,0 +1,115 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    VALID_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.hierarchical_logger import hwarn
+
+
+class EXAMSMultilingualScenario(Scenario):
+    """EXAMS: A Multi-subject High School Examinations Dataset
+
+    EXAMS is a benchmark dataset for multilingual and cross-lingual
+    question answering from high school examinations. It consists of
+    more than 24,000 high-quality high school exam questions in 16
+    languages, covering 8 language families and 24 school subjects
+    from Natural Sciences and Social Sciences, among others.
+
+    - https://huggingface.co/datasets/mhardalov/exams
+    - https://aclanthology.org/2020.emnlp-main.438/
+
+    Note: Some dataset rows have the value '@' in the `answerKey` column.
+    These rows will be ignored.
+
+    ```
+    @inproceedings{hardalov-etal-2020-exams,
+        title = "{EXAMS}: A Multi-subject High School Examinations Dataset for Cross-lingual and Multilingual Question Answering",
+        author = "Hardalov, Momchil  and
+        Mihaylov, Todor  and
+        Zlatkova, Dimitrina  and
+        Dinkov, Yoan  and
+        Koychev, Ivan  and
+        Nakov, Preslav",
+        editor = "Webber, Bonnie  and
+        Cohn, Trevor  and
+        He, Yulan  and
+        Liu, Yang",
+        booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
+        month = nov,
+        year = "2020",
+        address = "Online",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2020.emnlp-main.438/",
+        doi = "10.18653/v1/2020.emnlp-main.438",
+        pages = "5427--5444",
+        abstract = "We propose EXAMS {--} a new benchmark dataset for cross-lingual and multilingual question answering for high school examinations. We collected more than 24,000 high-quality high school exam questions in 16 languages, covering 8 language families and 24 school subjects from Natural Sciences and Social Sciences, among others.EXAMS offers unique fine-grained evaluation framework across multiple languages and subjects, which allows precise analysis and comparison of the proposed models. We perform various experiments with existing top-performing multilingual pre-trained models and show that EXAMS offers multiple challenges that require multilingual knowledge and reasoning in multiple domains. We hope that EXAMS will enable researchers to explore challenging reasoning and knowledge transfer methods and pre-trained models for school question answering in various languages which was not possible by now. The data, code, pre-trained models, and evaluation are available at http://github.com/mhardalov/exams-qa."
+    }```
+    """  # noqa: E501
+
+    name = "exams_multilingual"
+    description = "EXAMS is a benchmark dataset for multilingual and cross-lingual question answering from high school examinations. "  # noqa: E501
+    tags = ["knowledge", "multiple_choice"]
+
+    CHOICES = ["A", "B", "C", "D", "E"]
+    HF_SPLIT_TO_HELM_SPLIT = {"train": TRAIN_SPLIT, "test": TEST_SPLIT, "validation": VALID_SPLIT}
+
+    def __init__(self, language: str, subject: str):
+        super().__init__()
+        self.language = language
+        self.subject = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "mhardalov/exams",
+            "multilingual",
+            revision="4ff10804abb3341f8815cacd778181177bba7edd",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            for row in dataset:
+                question = row["question"]
+                question_info = row["info"]
+                if self.subject != "all" and question_info["subject"] != self.subject:
+                    continue
+                if self.language != "all" and question_info["language"] != self.language:
+                    continue
+                input = Input(text=question["stem"])
+                references: List[Reference] = []
+                if row["answerKey"] not in self.CHOICES:
+                    hwarn(f"Invalid value in answerKey column in row: {row}")
+                    continue
+                correct_choice_index = ord(row["answerKey"]) - ord("A")
+                for choice_index, choice_text in enumerate(question["choices"]["text"]):
+                    references.append(
+                        Reference(
+                            output=Output(text=choice_text),
+                            tags=[CORRECT_TAG] if choice_index == correct_choice_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=row["id"],
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/financial_phrasebank_scenario.py

@@ -2,6 +2,7 @@ import os
 import random
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -92,3 +94,22 @@ Possible labels:\n1. positive\n2. neutral\n3. negative"""  # noqa: E501
             )
             instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="financial_phrasebank",
+            display_name="Financial Phrasebank (Sentiment Classification)",
+            short_display_name=None,
+            description="A sentiment classification benchmark based on the dataset from Good Debt or "
+            "Bad Debt - Detecting Semantic Orientations in Economic Texts [(Malo et al., "
+            "2013)](https://arxiv.org/abs/1307.5336).",
+            taxonomy=TaxonomyInfo(
+                task="sentiment analysis",
+                what="phrases from financial news texts and company press releases",
+                when="before 2013",
+                who="annotators with adequate business education background",
+                language="English",
+            ),
+            main_metric="classification_weighted_f1",
+            main_split="test",
+        )

helm/benchmark/scenarios/gold_commodity_news_scenario.py

@@ -6,6 +6,7 @@ from typing import List
 
 import pandas as pd
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.runner import TRAIN_SPLIT
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
@@ -16,6 +17,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Scenario,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -122,3 +124,22 @@ class GoldCommodityNewsScenario(Scenario):
         for train_index in train_indexes:
             instances[train_index] = dataclasses.replace(instances[train_index], split=TRAIN_SPLIT)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="gold_commodity_news",
+            display_name="Gold Commodity News",
+            short_display_name=None,
+            description="A classification benchmark based on a dataset of human-annotated gold "
+            "commodity news headlines ([Sinha & Khandait, "
+            "2019](https://arxiv.org/abs/2009.04202)).",
+            taxonomy=TaxonomyInfo(
+                task="text classification",
+                what="gold commodity news headlines",
+                when="2000-2019",
+                who="financial journalists",
+                language="English",
+            ),
+            main_metric="classification_weighted_f1",
+            main_split="test",
+        )

helm/benchmark/scenarios/gpqa_scenario.py

@@ -2,6 +2,7 @@ import datasets
 import os
 import random
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -78,3 +80,19 @@ class GPQAScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="GPQA",
+            description=self.description,
+            main_metric="chain_of_thought_correctness",
+            main_split="test",
+            taxonomy=TaxonomyInfo(
+                task="question answering",
+                what="complex questions across various disciplines",
+                who="domain experts",
+                when="2024",
+                language="English",
+            ),
+        )

helm/benchmark/scenarios/grammar_scenario.py

@@ -1,6 +1,7 @@
 from typing import List
 
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, ScenarioMetadata
 from helm.benchmark.scenarios.grammar import read_grammar, generate_derivations, Derivation, get_values, get_tags
 
 
@@ -41,3 +42,21 @@ class GrammarScenario(Scenario):
         instances: List[Instance] = list(map(derivation_to_instance, derivations))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="grammar",
+            display_name="Best ChatGPT Prompts",
+            short_display_name="Best ChatGPT Prompts",
+            description="A list of “best ChatGPT prompts to power your workflow” summarized by "
+            "[GRIDFITI](https://gridfiti.com/best-chatgpt-prompts/).",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Instructions for LLMs",
+                when="2023",
+                who="Gridfiti Staff",
+                language="English",
+            ),
+            main_metric="Helpfulness",
+            main_split="test",
+        )

helm/benchmark/scenarios/gsm_scenario.py

@@ -2,6 +2,7 @@ import json
 import os
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -65,3 +67,16 @@ class GSM8KScenario(Scenario):
                         ),
                     )
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="gsm",
+            display_name="GSM8K (Grade school math word problems)",
+            short_display_name="GSM8K",
+            description="The grade school math word problems dataset (GSM8K) for testing mathematical "
+            "reasoning on grade-school math problems [(Cobbe et al., "
+            "2021)](https://arxiv.org/pdf/2110.14168.pdf).",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="exact_match_indicator",
+            main_split="test",
+        )

helm/benchmark/scenarios/headqa_scenario.py

@@ -3,6 +3,7 @@ from typing import List, Optional
 
 from datasets import DatasetDict, load_dataset
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     TEST_SPLIT,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
     Reference,
     Scenario,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -134,3 +136,23 @@ class HeadQAScenario(Scenario):
                     )
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="head_qa",
+            display_name="HeadQA",
+            description="HeadQA is a benchmark consisting of biomedical multiple-choice questions "
+            "intended to evaluate a model's medical knowledge and reasoning. Each instance "
+            "presents a clinical or scientific question with four answer options, requiring "
+            "the model to select the most appropriate answer [(Vilares et al., "
+            "2019)](https://arxiv.org/abs/1906.04701).",
+            taxonomy=TaxonomyInfo(
+                task="Question answering",
+                what="Medical knowledge testing",
+                when="Any",
+                who="Medical student, Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )