crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/scenarios/exams_multilingual_scenario.py

@@ -0,0 +1,115 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    VALID_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.hierarchical_logger import hwarn
+
+
+class EXAMSMultilingualScenario(Scenario):
+    """EXAMS: A Multi-subject High School Examinations Dataset
+
+    EXAMS is a benchmark dataset for multilingual and cross-lingual
+    question answering from high school examinations. It consists of
+    more than 24,000 high-quality high school exam questions in 16
+    languages, covering 8 language families and 24 school subjects
+    from Natural Sciences and Social Sciences, among others.
+
+    - https://huggingface.co/datasets/mhardalov/exams
+    - https://aclanthology.org/2020.emnlp-main.438/
+
+    Note: Some dataset rows have the value '@' in the `answerKey` column.
+    These rows will be ignored.
+
+    ```
+    @inproceedings{hardalov-etal-2020-exams,
+        title = "{EXAMS}: A Multi-subject High School Examinations Dataset for Cross-lingual and Multilingual Question Answering",
+        author = "Hardalov, Momchil  and
+        Mihaylov, Todor  and
+        Zlatkova, Dimitrina  and
+        Dinkov, Yoan  and
+        Koychev, Ivan  and
+        Nakov, Preslav",
+        editor = "Webber, Bonnie  and
+        Cohn, Trevor  and
+        He, Yulan  and
+        Liu, Yang",
+        booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
+        month = nov,
+        year = "2020",
+        address = "Online",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2020.emnlp-main.438/",
+        doi = "10.18653/v1/2020.emnlp-main.438",
+        pages = "5427--5444",
+        abstract = "We propose EXAMS {--} a new benchmark dataset for cross-lingual and multilingual question answering for high school examinations. We collected more than 24,000 high-quality high school exam questions in 16 languages, covering 8 language families and 24 school subjects from Natural Sciences and Social Sciences, among others.EXAMS offers unique fine-grained evaluation framework across multiple languages and subjects, which allows precise analysis and comparison of the proposed models. We perform various experiments with existing top-performing multilingual pre-trained models and show that EXAMS offers multiple challenges that require multilingual knowledge and reasoning in multiple domains. We hope that EXAMS will enable researchers to explore challenging reasoning and knowledge transfer methods and pre-trained models for school question answering in various languages which was not possible by now. The data, code, pre-trained models, and evaluation are available at http://github.com/mhardalov/exams-qa."
+    }```
+    """  # noqa: E501
+
+    name = "exams_multilingual"
+    description = "EXAMS is a benchmark dataset for multilingual and cross-lingual question answering from high school examinations. "  # noqa: E501
+    tags = ["knowledge", "multiple_choice"]
+
+    CHOICES = ["A", "B", "C", "D", "E"]
+    HF_SPLIT_TO_HELM_SPLIT = {"train": TRAIN_SPLIT, "test": TEST_SPLIT, "validation": VALID_SPLIT}
+
+    def __init__(self, language: str, subject: str):
+        super().__init__()
+        self.language = language
+        self.subject = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "mhardalov/exams",
+            "multilingual",
+            revision="4ff10804abb3341f8815cacd778181177bba7edd",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            for row in dataset:
+                question = row["question"]
+                question_info = row["info"]
+                if self.subject != "all" and question_info["subject"] != self.subject:
+                    continue
+                if self.language != "all" and question_info["language"] != self.language:
+                    continue
+                input = Input(text=question["stem"])
+                references: List[Reference] = []
+                if row["answerKey"] not in self.CHOICES:
+                    hwarn(f"Invalid value in answerKey column in row: {row}")
+                    continue
+                correct_choice_index = ord(row["answerKey"]) - ord("A")
+                for choice_index, choice_text in enumerate(question["choices"]["text"]):
+                    references.append(
+                        Reference(
+                            output=Output(text=choice_text),
+                            tags=[CORRECT_TAG] if choice_index == correct_choice_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=row["id"],
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/grammar.py

@@ -2,7 +2,7 @@ from collections import defaultdict
 from dataclasses import dataclass, field, replace
 from functools import cached_property
 from typing import List, Optional
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hwarn
 
 import dacite
 import re
@@ -111,7 +111,7 @@ def validate_grammar(grammar: Grammar):
             # Make sure all categories are defined
             for category in expansion.categories:
                 if category not in grammar.category_to_rules:
-                    hlog(f"WARNING: Category {category} is not defined")
+                    hwarn(f"Category {category} is not defined")
 
 
 def read_grammar(path: str) -> Grammar:

helm/benchmark/scenarios/headqa_scenario.py

@@ -57,7 +57,12 @@ class HeadQAScenario(Scenario):
     SKIP_TEXTQA: bool = False
 
     name = "head_qa"
-    description = "A collection of biomedical multiple-choice questions for testing medical knowledge."
+    description = (
+        "HeadQA is a benchmark consisting of biomedical multiple-choice questions intended to"
+        "evaluate a model's medical knowledge and reasoning. Each instance presents a clinical"
+        "or scientific question with four answer options, requiring the model to select the most"
+        "appropriate answer."
+    )
     tags = ["question_answering", "biomedical", "medicine"]
 
     def __init__(self, language: str = "en", category: Optional[str] = None):

helm/benchmark/scenarios/healthqa_br_scenario.py

@@ -0,0 +1,80 @@
+from typing import Any, List
+import re
+from pathlib import Path
+from datasets import load_dataset
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class HEALTHQA_BR_Scenario(Scenario):
+    """
+    HealthQA-BR is a large-scale benchmark designed to evaluate the clinical knowledge of Large Language Models (LLMs)
+    within the Brazilian Unified Health System (SUS) context. It comprises 5,632 multiple-choice questions sourced from
+    nationwide licensing exams and residency tests, reflecting real challenges faced by Brazil's public health sector.
+    Unlike benchmarks focused on the U.S. medical landscape, HealthQA-BR targets the Brazilian healthcare ecosystem,
+    covering a wide range of medical specialties and interdisciplinary professions such as nursing, dentistry,
+    psychology, social work, pharmacy, and physiotherapy. This comprehensive approach enables a detailed assessment
+    of AI models’ ability to collaborate effectively in the team-based patient care typical of SUS.
+    """
+
+    name = "healthqa_br"
+    description = "MQA benchmark with questions from Brazilian entrance exams"
+    tags = ["knowledge", "multiple_choice", "pt-br"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the raw data and read all the dialogues
+        dataset: Any
+        # Read all the instances
+        instances: List[Instance] = []
+        cache_dir = str(Path(output_path) / "data")
+
+        dataset = load_dataset("Larxel/healthqa-br", cache_dir=cache_dir)
+        for example in dataset["train"]:
+            question_choices = example["question"]
+            answer = example["answer"].strip().upper()
+
+            # Separate the question statement from the alternatives
+            question_text, choices_text = self.split_question_and_choices(question_choices)
+
+            # Extract alternatives from text choices_text
+            pattern = r"'([A-Z])':\s*'([^']+)'"
+            matches = re.findall(pattern, choices_text)
+            answers_dict = {label: text for label, text in matches}
+
+            if answer not in answers_dict:
+                continue
+
+            correct_answer_text = answers_dict[answer]
+
+            def answer_to_reference(answer: str) -> Reference:
+                return Reference(Output(text=answer), tags=[CORRECT_TAG] if correct_answer_text == answer else [])
+
+            instance = Instance(
+                input=Input(text=question_text),
+                split=TEST_SPLIT,
+                references=[answer_to_reference(text) for text in answers_dict.values()],
+            )
+            instances.append(instance)
+        return instances
+
+    def split_question_and_choices(self, full_text: str):
+        # Search for the first occurrence of the alternative pattern
+        match = re.search(r"\n'[A-Z]':\s*'.+", full_text)
+        if match:
+            # Everything before the alternatives
+            question_part = full_text[: match.start()].strip()
+            # All of the alternatives (from match to end)
+            choices_part = full_text[match.start() :].strip()
+        else:
+            # If you don't find a pattern, consider everything as a question, and no alternative.
+            question_part = full_text.strip()
+            choices_part = ""
+
+        return question_part, choices_part

helm/benchmark/scenarios/infinite_bench_en_mc_scenario.py

@@ -0,0 +1,90 @@
+import os
+import re
+from typing import List
+
+from datasets import load_dataset, Features, Value, Sequence, Dataset
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    Reference,
+    Output,
+    CORRECT_TAG,
+    TEST_SPLIT,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class InfiniteBenchEnMCScenario(Scenario):
+    """InfiniteBench En.MC
+
+    InfiniteBench is a benchmark tailored for evaluating the capabilities of language models to process,
+    understand, and reason over long contexts (100k+ tokens). InfiniteBench En.MC is a subset of
+    InfiniteBench that requires models to perform multiple-choice question answering on questions that necessitate
+    long-range dependency and reasoning, beyond simple short passage retrieval.
+    """
+
+    name = "infinite_bench_en_mc"
+    description = "∞Bench En.MC is a multiple-choice question answering task that necessitates long-range dependency and reasoning. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))"  # noqa: E501
+    tags = ["question_answering"]
+
+    def __init__(self, max_num_words: int):
+        self.max_num_words = max_num_words
+        super().__init__()
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Get InfiniteBench from HuggingFace
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+
+        # Define the features schema
+        ft = Features(
+            {
+                "id": Value("int64"),
+                "context": Value("string"),
+                "input": Value("string"),
+                "answer": Sequence(Value("string")),
+                "options": Sequence(Value("string")),
+            }
+        )
+
+        # Load the dataset with the specified features
+        dataset = load_dataset(
+            "xinrongzhang2022/InfiniteBench",
+            split="longbook_choice_eng",
+            features=ft,
+            revision="90f0394333616266d9fe85824ceaf505093cbaa5",
+        )
+
+        assert isinstance(dataset, Dataset)
+
+        def count_words(text: str) -> int:
+            return len(re.split(r"\s+", text.strip()))
+
+        dataset = dataset.filter(
+            lambda example: count_words(example["context"])
+            + count_words(example["input"])
+            + sum(count_words(option) for option in example["options"])
+            <= self.max_num_words
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row in dataset:
+            assert len(row["answer"]) == 1
+            id = row["id"]
+            input = Input(text=row["context"] + "\n\n" + row["input"])
+            references = [
+                Reference(Output(text=option), tags=[CORRECT_TAG] if option == row["answer"][0] else [])
+                for option in row["options"]
+            ]
+            instance = Instance(
+                id=id,
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/infinite_bench_en_qa_scenario.py

@@ -0,0 +1,85 @@
+import os
+import re
+from typing import List
+
+from datasets import load_dataset, Features, Value, Sequence, Dataset
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    Reference,
+    Output,
+    CORRECT_TAG,
+    TEST_SPLIT,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class InfiniteBenchEnQAScenario(Scenario):
+    """InfiniteBench En.QA
+
+    InfiniteBench is a benchmark tailored for evaluating the capabilities of language models to process,
+    understand, and reason over long contexts (100k+ tokens). InfiniteBench En.QA is a subset of
+    InfiniteBench that requires models to perform open-form question answering on questions that necessitate
+    long-range dependency and reasoning, beyond simple short passage retrieval.
+    """
+
+    name = "infinite_bench_en_qa"
+    description = "∞Bench En.QA is an open-ended question answering task that necessitates long-range dependency and reasoning.  ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))"  # noqa: E501
+    tags = ["question_answering"]
+
+    def __init__(self, max_num_words: int):
+        self.max_num_words = max_num_words
+        super().__init__()
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Get InfiniteBench from HuggingFace
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+
+        # Define the features schema
+        ft = Features(
+            {
+                "id": Value("int64"),
+                "context": Value("string"),
+                "input": Value("string"),
+                "answer": Sequence(Value("string")),
+                "options": Sequence(Value("string")),
+            }
+        )
+
+        # Load the dataset with the specified features
+        dataset = load_dataset(
+            "xinrongzhang2022/InfiniteBench",
+            split="longbook_qa_eng",
+            features=ft,
+            revision="90f0394333616266d9fe85824ceaf505093cbaa5",
+        )
+
+        assert isinstance(dataset, Dataset)
+
+        def count_words(text: str) -> int:
+            return len(re.split(r"\s+", text.strip()))
+
+        dataset = dataset.filter(
+            lambda example: count_words(example["context"])
+            + count_words(example["input"])
+            + sum(count_words(option) for option in example["options"])
+            <= self.max_num_words
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row in dataset:
+            id = row["id"]
+            input = Input(text=row["context"] + "\n\n" + row["input"])
+            instance = Instance(
+                id=id,
+                input=input,
+                references=[Reference(Output(text=row["answer"][0]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/{infinite_bench_sum_scenario.py → infinite_bench_en_sum_scenario.py}

@@ -14,21 +14,19 @@ from helm.benchmark.scenarios.scenario import (
 from helm.common.general import ensure_directory_exists
 
 
-class InfiniteBenchSumScenario(Scenario):
-    """InfiniteBench Sum
+class InfiniteBenchEnSumScenario(Scenario):
+    """InfiniteBench En.Sum
 
     InfiniteBench is a benchmark tailored for evaluating the capabilities of language models to process,
-    understand, and reason over super long contexts (100k+ tokens). InfiniteBench Sum is a subset of
-    InfiniteBench that requires models to generate a concise summary of the novel. The subset is referred
-    to as "En.Sum" in the original paper.
+    understand, and reason over super long contexts (100k+ tokens). InfiniteBench En.Sum is a subset of
+    InfiniteBench that requires models to generate a concise summary of the novel.
     """
 
-    name = "infinite_bench_sum"
-    description = "Summarize a novel from InfiniteBench"
+    name = "infinite_bench_en_sum"
+    description = "∞Bench En.Sum is a summarization task that requires generating a concise summary of a novel. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))"  # noqa: E501
     tags = ["summarization"]
 
-    def __init__(self, min_num_words: int, max_num_words: int):
-        self.min_num_words = min_num_words
+    def __init__(self, max_num_words: int):
         self.max_num_words = max_num_words
         super().__init__()
 
@@ -61,9 +59,9 @@ class InfiniteBenchSumScenario(Scenario):
         def count_words(text: str) -> int:
             return len(re.split(r"\s+", text.strip()))
 
-        dataset = dataset.map(
-            lambda example: {"prompt_wc": count_words(example["context"]) + count_words(example["input"])}
-        ).filter(lambda example: self.min_num_words <= example["prompt_wc"] <= self.max_num_words)
+        dataset = dataset.filter(
+            lambda example: count_words(example["context"]) + count_words(example["input"]) <= self.max_num_words
+        )
 
         # Read all instances
         instances: List[Instance] = []
@@ -75,7 +73,6 @@ class InfiniteBenchSumScenario(Scenario):
                 input=input,
                 references=[Reference(Output(text=row["answer"][0]), tags=[CORRECT_TAG])],
                 split=TEST_SPLIT,
-                extra_data={"word_count": row["prompt_wc"]},
             )
             instances.append(instance)
 

helm/benchmark/scenarios/kpi_edgar_scenario.py

@@ -0,0 +1,151 @@
+import os
+from typing import List, Dict
+import json
+import re
+
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class KPIEDGARScenario(Scenario):
+    """A financial named entity recognition (NER) scenario based on KPI-EDGAR (T. Deußer et al., 2022).
+
+    This scenario has been modified from the paper. The original paper has 12 entity types and requires the model
+    to extract pairs of related entities. This scenario only use four named entity types (kpi, cy, py, py1) and only
+    requires the model to extract individual entities.
+
+    Paper:
+    T. Deußer et al.,
+    “KPI-EDGAR: A Novel Dataset and Accompanying Metric for Relation Extraction from Financial Documents.” 2022.
+    https://arxiv.org/abs/2210.09163
+
+    Prompt format:
+
+    ```
+    Context: {Sentence}
+    Task: Extract key performance indicators (KPIs) and values from the above text. Also, specify one of the following categories to each of the extracted KPIs and values in brackets.
+    kpi: Key Performance Indicators expressible in numerical and monetary value, cy: Current Year monetary value, py: Prior Year monetary value, py1: Two Year Past Value.
+    Answer:
+    ```
+
+    Example input:
+
+    ```
+    Context: The following table summarizes our total share-based compensation expense and excess tax benefits recognized : As of December 28 , 2019 , there was $ 284 million of total unrecognized compensation cost related to nonvested share-based compensation grants .
+    Task: Extract key performance indicators (KPIs) and values from the above text. Also, specify one of the following categories to each of the extracted KPIs and values in brackets.
+    kpi: Key Performance Indicators expressible in numerical and monetary value, cy: Current Year monetary value, py: Prior Year monetary value, py1: Two Year Past Value.
+    Answer:
+    ```
+
+    Example reference:
+    ```
+    284 [cy], total unrecognized compensation cost [kpi]
+    ```"""  # noqa: E501
+
+    name = "kpi_edgar"
+    description = "Named Entity Recognition from financial documents."
+    tags = ["named_entity_recognition", "finance"]
+
+    TAG_DICT = {
+        "kpi": "Key Performance Indicators expressible in numerical and monetary value",
+        "cy": "Current Year monetary value",
+        "py": "Prior Year monetary value",
+        "py1": "Two Year Past Value",
+    }
+    TAG_DESCRIPTIONS = ", ".join(["%s: %s" % (key, val) for (key, val) in TAG_DICT.items()]) + "."
+    TAG_PAREN_RE = (r"\[", r"\]")
+    TAG_PAREN = tuple((e.strip("\\") for e in TAG_PAREN_RE))
+    TAG_PAREN_ESC = ("(", ")")
+    DATASET_SPLIT_TO_HELM_SPLIT = {"train": TRAIN_SPLIT, "valid": VALID_SPLIT, "test": TEST_SPLIT}
+    JSON_URL = "https://raw.githubusercontent.com/tobideusser/kpi-edgar/2ec7084dcd55b4979bbe288d4aa1e962c685c9ab/data/kpi_edgar.json"  # noqa: E501
+    JSON_FILENAME = "kpi_edgar.json"
+
+    @staticmethod
+    def get_sentences(dataset: List[Dict]) -> List[Dict]:
+        return [
+            sentence
+            for document in dataset
+            for segment in document["segments"]
+            for sentence in segment["sentences"] or []
+        ]
+
+    @staticmethod
+    def escape_parenthesis(text: str) -> str:
+        tmp0 = re.sub(KPIEDGARScenario.TAG_PAREN_RE[0], KPIEDGARScenario.TAG_PAREN_ESC[0], text)
+        tmp1 = re.sub(KPIEDGARScenario.TAG_PAREN_RE[1], KPIEDGARScenario.TAG_PAREN_ESC[1], tmp0)
+        return tmp1
+
+    @staticmethod
+    def get_output_text(
+        words: List[str],
+        annotations: List[Dict],
+    ) -> str:
+        # def get_entity_for_annotation(words: List[str], annotation: Dict) -> str
+        entities: List[str] = []
+        for annotation in annotations:
+            annotation_type = annotation["type_"]
+            if annotation_type not in KPIEDGARScenario.TAG_DICT:
+                continue
+            start_idx = annotation["start"]
+            end_idx = annotation["end"]
+            annotated_words = words[start_idx:end_idx]
+            phrase = KPIEDGARScenario.escape_parenthesis(" ".join(annotated_words))
+            entities.append(
+                "%s %s%s%s" % (phrase, KPIEDGARScenario.TAG_PAREN[0], annotation_type, KPIEDGARScenario.TAG_PAREN[1])
+            )
+
+        return ", ".join(entities)
+
+    @staticmethod
+    def sentences_to_instances(sentences: List[Dict]) -> List[Instance]:
+        instances: List[Instance] = []
+        for sentence in sentences:
+            dataset_split: str = sentence["split_type"]
+            if dataset_split is None:
+                continue
+            split = KPIEDGARScenario.DATASET_SPLIT_TO_HELM_SPLIT[dataset_split]
+
+            words: List[str] = [word_dict["value"] for word_dict in sentence["words"]]
+            passage = KPIEDGARScenario.escape_parenthesis(" ".join(words))
+            input_text = (
+                "Context: %s\n"
+                "Task: Extract key performance indicators (KPIs) and values from the above text. Also, specify one of the following categories to each of the extracted KPIs and values in brackets.\n"  # noqa: E501
+                "%s" % (passage, KPIEDGARScenario.TAG_DESCRIPTIONS)
+            )
+
+            annotations = sentence["entities_anno"]
+            output_text = KPIEDGARScenario.get_output_text(words, annotations)
+            if not output_text:
+                continue
+
+            instances.append(
+                Instance(
+                    input=Input(text=input_text),
+                    references=[Reference(Output(text=output_text), tags=[CORRECT_TAG])],
+                    split=split,
+                )
+            )
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = os.path.join(output_path, "data")
+        ensure_directory_exists(data_path)
+
+        base_url = self.JSON_URL
+        dataset_file_name = self.JSON_FILENAME
+        target_path = os.path.join(data_path, dataset_file_name)
+        ensure_file_downloaded(source_url=base_url, target_path=target_path)
+
+        with open(target_path, "r") as f:
+            raw_dataset = json.load(f)
+        return KPIEDGARScenario.sentences_to_instances(KPIEDGARScenario.get_sentences(raw_dataset))