crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/enem_challenge_scenario.py

@@ -0,0 +1,77 @@
+from typing import List, Any
+from pathlib import Path
+from datasets import load_dataset
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
+
+
+class ENEMChallengeScenario(Scenario):
+    """
+    The Exame Nacional do Ensino Médio (ENEM) is an advanced High-School level exam widely applied
+    every year by the Brazilian government to students that wish to undertake a University degree.
+
+    The questions are about all types of intelectual fields and they are divided into four groups
+    that are named as: Humanities, Languages, Sciences and Mathematics.
+
+    This scenario is based on the exams that were applied throughout the years of 2009 and 2023.
+
+    The dataset can be found in this link: https://huggingface.co/datasets/eduagarcia/enem_challenge
+    """
+
+    name = "enem_challenge"
+    description = "ENEM Challenge dataset"
+    tags = ["knowledge", "multiple_choice", "pt-br"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the raw data and read all the dialogues
+        dataset: Any
+        # Read all the instances
+        instances: List[Instance] = []
+        cache_dir = str(Path(output_path) / "data")
+
+        dataset = load_dataset("eduagarcia/enem_challenge", cache_dir=cache_dir)
+        for example in dataset["train"]:
+            question = example["question"]
+            choices = example["choices"]
+            answer = example["answerKey"]
+            # Skipping every canceled question!
+            if answer == "ANULADO":
+                continue
+            answers_dict = dict(zip(choices["label"], choices["text"]))
+            correct_answer = answers_dict[answer]
+
+            def answer_to_reference(answer: str) -> Reference:
+                return Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+
+            instance = Instance(
+                input=Input(text=question), split=TEST_SPLIT, references=list(map(answer_to_reference, choices["text"]))
+            )
+            instances.append(instance)
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="enem_challenge",
+            display_name="ENEM Challenge",
+            short_display_name=None,
+            description="ENEM Challenge",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="general academic subjects",
+                when="between 2009 and 2023",
+                who="brazilian ministry of education",
+                language="Portuguese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/entity_data_imputation_scenario.py

@@ -3,9 +3,21 @@ import pandas as pd
 from pathlib import Path
 from typing import List, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.hierarchical_logger import hlog
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class EntityDataImputationScenario(Scenario):
@@ -41,8 +53,14 @@ class EntityDataImputationScenario(Scenario):
     def __init__(self, dataset: str, seed: int = 1234):
         super().__init__()
         self.datasets_paths = {
-            "Buy": "https://dbs.uni-leipzig.de/file/Abt-Buy.zip",
-            "Restaurant": "https://www.cs.utexas.edu/users/ml/riddle/data/restaurant.tar.gz",
+            "Buy": (
+                "https://storage.googleapis.com/crfm-helm-public/source_datasets/scenarios/"
+                "entity_data_imputation/Abt-Buy.zip"
+            ),
+            "Restaurant": (
+                "https://storage.googleapis.com/crfm-helm-public/source_datasets/scenarios/"
+                "entity_data_imputation/restaurant.tar.gz"
+            ),
         }
         # Columns to impute
         self.datasets_impute_col = {
@@ -144,3 +162,15 @@ class EntityDataImputationScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="entity_data_imputation",
+            display_name="Data imputation",
+            description="Scenario from [Mei et al. "
+            "(2021)](https://ieeexplore.ieee.org/document/9458712/) that tests the ability "
+            "to impute missing entities in a data table.",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/entity_matching_scenario.py

@@ -2,10 +2,22 @@ import pandas as pd
 from pathlib import Path
 from typing import Dict, List, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.hierarchical_logger import hlog
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
-from .entity_matching_scenario_fixed_random_state import set_fixed_random_state_for_dataset
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
+from helm.benchmark.scenarios.entity_matching_scenario_fixed_random_state import set_fixed_random_state_for_dataset
 
 
 class EntityMatchingScenario(Scenario):
@@ -145,3 +157,15 @@ class EntityMatchingScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="entity_matching",
+            display_name="Entity matching",
+            description="Scenario from Magellan [(Konda et al., "
+            "2016)](https://dl.acm.org/doi/10.14778/3007263.3007314) that tests the ability "
+            "to determine if two entities match.",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/ewok_scenario.py

@@ -0,0 +1,116 @@
+import datasets
+import os
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    Input,
+    Output,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class EWoKScenario(Scenario):
+    """Elements of World Knowledge (EWoK)
+
+    Elements of World Knowledge (EWoK) is a framework for evaluating world modeling in
+    language models by testing their ability to use knowledge of a concept to match a
+    target text with a plausible/implausible context. EWoK targets specific concepts
+    from multiple knowledge domains known to be vital for world modeling in humans.
+    Domains range from social interactions (help/hinder) to spatial relations (left/right).
+    Both, contexts and targets are minimal pairs. Objects, agents, and locations in the items
+    can be flexibly filled in enabling easy generation of multiple controlled datasets.
+
+    EWoK-CORE-1.0 is a dataset of 4,374 items covering 11 world knowledge domains."""
+
+    name = "ewok"
+    description = (
+        "Elements of World Knowledge (EWoK) is a benchmark for evaluating world modeling by testing their ability to "
+        "use knowledge of a concept to match a target text with a plausible/implausible context."
+    )
+    tags = ["world knowledge"]
+
+    DOMAINS = [
+        "agent_properties",
+        "material_dynamics",
+        "material_properties",
+        "physical_dynamics",
+        "physical_interactions",
+        "physical_relations",
+        "quantitative_properties",
+        "social_interactions",
+        "social_properties",
+        "social_relations",
+        "spatial_relations",
+    ]
+
+    def __init__(self, domain: str = "all"):
+        super().__init__()
+        if domain != "all" and domain not in EWoKScenario.DOMAINS:
+            raise Exception(f"Unknown domain '{domain}', valid domains are {EWoKScenario.DOMAINS}")
+        self.domain = domain.replace("_", "-")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        next_instance_index = 0
+
+        def generate_instance_id() -> str:
+            nonlocal next_instance_index
+            instance_id = f"id{next_instance_index}"
+            next_instance_index += 1
+            return instance_id
+
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+
+        # TODO: Switch this to the production dataset when available.
+        dataset = datasets.load_dataset(
+            "ewok-core/ewok-core-1.0",
+            split="test",
+            revision="34d912a608066c92e2990a0328ffc3bd9a716042",
+            cache_dir=cache_dir,
+        )
+
+        instances: List[Instance] = []
+        for row in dataset:
+            contexts = [row["Context1"], row["Context2"]]
+            targets = [row["Target1"], row["Target2"]]
+            # References are category ID, followed by level 2, 3 and 4 category names.
+            for target_index, target in enumerate(targets):
+                input = Input(text=target)
+                references = [
+                    Reference(output=Output(text=context), tags=[CORRECT_TAG] if context_index == target_index else [])
+                    for context_index, context in enumerate(contexts)
+                ]
+                instance = Instance(id=generate_instance_id(), input=input, references=references, split=TEST_SPLIT)
+                # Filtering by domain after generate instance IDs,
+                # so that instance IDs for an item is invariant regardless of domain filtering.
+                if self.domain == "all" or self.domain == row["Domain"]:
+                    instances.append(instance)
+        instances.extend(
+            [
+                Instance(
+                    id=generate_instance_id(),
+                    input=Input(text="I drew a ball from the bag."),
+                    references=[
+                        Reference(output=Output(text="The bag is full of blocks."), tags=[]),
+                        Reference(output=Output(text="The bag is full of balls."), tags=[CORRECT_TAG]),
+                    ],
+                    split=TRAIN_SPLIT,
+                ),
+                Instance(
+                    id=generate_instance_id(),
+                    input=Input(text="The boy chose to eat a cookie."),
+                    references=[
+                        Reference(output=Output(text="The boy likes cookies."), tags=[CORRECT_TAG]),
+                        Reference(output=Output(text="The boy does not like cookies."), tags=[]),
+                    ],
+                    split=TRAIN_SPLIT,
+                ),
+            ]
+        )
+        return instances

helm/benchmark/scenarios/exams_multilingual_scenario.py

@@ -0,0 +1,115 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    VALID_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.hierarchical_logger import hwarn
+
+
+class EXAMSMultilingualScenario(Scenario):
+    """EXAMS: A Multi-subject High School Examinations Dataset
+
+    EXAMS is a benchmark dataset for multilingual and cross-lingual
+    question answering from high school examinations. It consists of
+    more than 24,000 high-quality high school exam questions in 16
+    languages, covering 8 language families and 24 school subjects
+    from Natural Sciences and Social Sciences, among others.
+
+    - https://huggingface.co/datasets/mhardalov/exams
+    - https://aclanthology.org/2020.emnlp-main.438/
+
+    Note: Some dataset rows have the value '@' in the `answerKey` column.
+    These rows will be ignored.
+
+    ```
+    @inproceedings{hardalov-etal-2020-exams,
+        title = "{EXAMS}: A Multi-subject High School Examinations Dataset for Cross-lingual and Multilingual Question Answering",
+        author = "Hardalov, Momchil  and
+        Mihaylov, Todor  and
+        Zlatkova, Dimitrina  and
+        Dinkov, Yoan  and
+        Koychev, Ivan  and
+        Nakov, Preslav",
+        editor = "Webber, Bonnie  and
+        Cohn, Trevor  and
+        He, Yulan  and
+        Liu, Yang",
+        booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
+        month = nov,
+        year = "2020",
+        address = "Online",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2020.emnlp-main.438/",
+        doi = "10.18653/v1/2020.emnlp-main.438",
+        pages = "5427--5444",
+        abstract = "We propose EXAMS {--} a new benchmark dataset for cross-lingual and multilingual question answering for high school examinations. We collected more than 24,000 high-quality high school exam questions in 16 languages, covering 8 language families and 24 school subjects from Natural Sciences and Social Sciences, among others.EXAMS offers unique fine-grained evaluation framework across multiple languages and subjects, which allows precise analysis and comparison of the proposed models. We perform various experiments with existing top-performing multilingual pre-trained models and show that EXAMS offers multiple challenges that require multilingual knowledge and reasoning in multiple domains. We hope that EXAMS will enable researchers to explore challenging reasoning and knowledge transfer methods and pre-trained models for school question answering in various languages which was not possible by now. The data, code, pre-trained models, and evaluation are available at http://github.com/mhardalov/exams-qa."
+    }```
+    """  # noqa: E501
+
+    name = "exams_multilingual"
+    description = "EXAMS is a benchmark dataset for multilingual and cross-lingual question answering from high school examinations. "  # noqa: E501
+    tags = ["knowledge", "multiple_choice"]
+
+    CHOICES = ["A", "B", "C", "D", "E"]
+    HF_SPLIT_TO_HELM_SPLIT = {"train": TRAIN_SPLIT, "test": TEST_SPLIT, "validation": VALID_SPLIT}
+
+    def __init__(self, language: str, subject: str):
+        super().__init__()
+        self.language = language
+        self.subject = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "mhardalov/exams",
+            "multilingual",
+            revision="4ff10804abb3341f8815cacd778181177bba7edd",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            for row in dataset:
+                question = row["question"]
+                question_info = row["info"]
+                if self.subject != "all" and question_info["subject"] != self.subject:
+                    continue
+                if self.language != "all" and question_info["language"] != self.language:
+                    continue
+                input = Input(text=question["stem"])
+                references: List[Reference] = []
+                if row["answerKey"] not in self.CHOICES:
+                    hwarn(f"Invalid value in answerKey column in row: {row}")
+                    continue
+                correct_choice_index = ord(row["answerKey"]) - ord("A")
+                for choice_index, choice_text in enumerate(question["choices"]["text"]):
+                    references.append(
+                        Reference(
+                            output=Output(text=choice_text),
+                            tags=[CORRECT_TAG] if choice_index == correct_choice_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=row["id"],
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/fin_qa_scenario.py

@@ -0,0 +1,139 @@
+import os
+import json
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    ScenarioMetadata,
+)
+
+
+DATASET_URL_PREFIX = "https://github.com/czyssrs/FinQA/raw/0f16e2867befa6840783e58be38c9efb9229d742/dataset/"
+INSTRUCTIONS = """Presented with a financial report consisting of textual contents and a structured table, given a question, generate the reasoning program in the domain specific langauge (DSL) that will be executed to get the answer.
+
+Respond with only the program in the DSL for the last question, without any preamble, elaboration, or working steps. Do not respond with anything that is not part of the DSL.
+
+The DSL consists of mathematical operations and table operations as executable programs. The program consists of a sequence of operations. Each operation takes a list of arguments.
+
+There are 6 mathematical operations: add, subtract, multiply, divide, greater, exp, and 4 table aggregation operations table-max, table-min, table-sum, table-average, that apply aggregation operations on table rows. The mathematical operations take arguments of either numbers from the given reports, or a numerical result from a previous step.
+
+The table operations take arguments of table row names. We use the special token #n to denote the result from the nth step.
+
+For example, in the example "divide(9413, 20.01), divide(8249, 9.48), subtract(#0, #1)", the program consists of 3 steps; The first and the second division steps take arguments from the table and the text, respectively, then the third step subtracts the results from the two previous steps.
+
+Definitions of all operations:
+
+[["Name", "Arguments", "Output", "Description"],
+["add", "number1, number2", "number", "add two numbers: number1 + number2"],
+["subtract", "number1, number2", "number", "subtract two numbers: number1 − number2"],
+["multiply", "number1, number2", "number", "multiply two numbers: number1 * number2"],
+["divide", "number1, number2", "number", "multiply two numbers: number1 / number2"],
+["exp", "number1, number2", "number", "exponential: number1 ^ number2"],
+["greater", "number1, number2", "bool", "comparison: number1 > number2"],
+["table-sum", "table header", "number", "the summation of one table row"],
+["table-average", "table header", "number", "the average of one table row"],
+["table-max", "table header", "number", "the maximum number of one table row"],
+["table-min", "table header", "number", "the minimum number of one table row"]]
+
+Answer with only the program, without any additional explanation.
+"""  # noqa: E501
+
+
+class FinQAScenario(Scenario):
+    """
+    FinQA is a question answering task over financial reports that requires robust numerical reasoning.
+
+    FinQA: A Dataset of Numerical Reasoning over Financial Data
+    Paper: https://arxiv.org/abs/2109.00122
+    Code: https://github.com/czyssrs/FinQA
+
+    Presented with a financial report consisting of textual contents and a structured table, given a question,
+    the task is togenerate the reasoning program in the domain specific langauge (DSL) that will be executed
+    to get the answer.
+
+    We add the sub-headers "Pre-table text", "Table", "Post-table text" to the input. Example:
+
+    ```
+    Pre-table text: printing papers net sales for 2006 decreased 3% ( 3 % ) from both 2005 and 2004 due principally...
+    [more lines]
+    Table: [["in millions", "2006", "2005", "2004"], ["sales", "$ 6930", "$ 7170", "$ 7135"], ["operating profit", "$ 677", "$ 473", "$ 508"]]
+    Post-table text: u.s .
+    uncoated papers net sales in 2006 were $ 3.5 billion , compared with $ 3.2 billion in 2005 and $ 3.3 billion in 2004 .
+    [more lines]
+    Question: brazilian paper sales represented what percentage of printing papers in 2005?
+    Program:
+    ```
+    """  # noqa: E501
+
+    name = "fin_qa"
+    description = "FinQA"
+    tags = ["question_answering", "financial"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path: str = os.path.join(output_path, "data")
+        ensure_directory_exists(data_path)
+        # Note: only train and test splits are used; dev split is not used
+        instances: List[Instance] = []
+        for split in [TRAIN_SPLIT, TEST_SPLIT]:
+            file_name = f"{split}.json"
+            target_path = os.path.join(data_path, file_name)
+            ensure_file_downloaded(
+                source_url=DATASET_URL_PREFIX + file_name,
+                target_path=target_path,
+            )
+            with open(target_path, "r") as f:
+                rows = json.load(f)
+                for row in rows:
+                    pre_text = "Pre-table text: " + "\n".join(row["pre_text"])
+                    table = "Table: " + json.dumps(row["table"])
+                    post_text = "Post-table text: " + "\n".join(row["post_text"])
+                    question = "Question: " + row["qa"]["question"]
+                    text = "\n".join([pre_text, table, post_text, question])
+                    references = [
+                        Reference(
+                            Output(text=str(row["qa"]["program"])),
+                            tags=[CORRECT_TAG],
+                        ),
+                        Reference(
+                            Output(text=str(row["qa"]["exe_ans"])),
+                            tags=[],
+                        ),
+                        Reference(
+                            Output(text=json.dumps(row["table"])),
+                            tags=[],
+                        ),
+                    ]
+                    instance: Instance = Instance(
+                        input=Input(text=text),
+                        references=references,
+                        split=split,
+                    )
+                    instances.append(instance)
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="fin_qa",
+            display_name="FinQA",
+            description="The FinQA benchmark for numeric reasoning over financial data, with question "
+            "answering pairs written by financial experts over financial reports [(Chen et "
+            "al., 2021)](https://arxiv.org/abs/2109.00122/).",
+            taxonomy=TaxonomyInfo(
+                task="question answering with numeric reasoning",
+                what="financial reports",
+                when="1999 to 2019",
+                who="financial experts",
+                language="English",
+            ),
+            main_metric="program_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/financebench_scenario.py

@@ -0,0 +1,74 @@
+import dataclasses
+import json
+import os
+import random
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class FinanceBenchScenario(Scenario):
+    """FinanceBench"""
+
+    name = "financebench"
+    description = "FinanceBench"
+    tags = ["finance"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        target_path = os.path.join(cache_dir, "financebench_open_source.jsonl")
+        url: str = (
+            "https://raw.githubusercontent.com/patronus-ai/financebench/d7beebe5e739e0b806ab4443c1b3e23f51804acf/data/financebench_open_source.jsonl"  # noqa: E501
+        )
+        ensure_file_downloaded(source_url=url, target_path=target_path)
+
+        instances: List[Instance] = []
+        with open(target_path) as f:
+            for line in f:
+                row = json.loads(line)
+                instance_id = row["financebench_id"]
+                question = row["question"]
+                answer = row["answer"]
+                evidence = row["evidence"][0]["evidence_text_full_page"]
+                input_text = f"Evidence: {evidence}\nQuestion: {question}"
+                input = Input(text=input_text)
+                references = [Reference(output=Output(text=answer), tags=[CORRECT_TAG])]
+                instance = Instance(id=instance_id, input=input, references=references, split=TEST_SPLIT)
+                instances.append(instance)
+        random.seed(0)
+        train_indexes = random.sample(list(range(len(instances))), k=10)
+        for train_index in train_indexes:
+            instances[train_index] = dataclasses.replace(instances[train_index], split=TRAIN_SPLIT)
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="financebench",
+            display_name="FinanceBench",
+            description="FinanceBench is a benchmark for open book financial question answering. It "
+            "comprises 10,231 questions about publicly traded companies, with corresponding "
+            "answers and evidence strings [(Islam et al., "
+            "2023)](https://arxiv.org/abs/2311.11944/).",
+            taxonomy=TaxonomyInfo(
+                task="question answering with numeric reasoning",
+                what="financial reports",
+                when="2015 to 2023",
+                who="financial experts",
+                language="English",
+            ),
+            main_metric="annotation_financebench_label_correct_answer",
+            main_split="test",
+        )