crfm-helm 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/scenarios/math_scenario.py

@@ -1,9 +1,20 @@
 import collections
+import os
 import typing
 from typing import Dict, List, Optional
 from datasets import load_dataset, DatasetDict
 
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 def remove_boxed(string: str) -> Optional[str]:
@@ -354,7 +365,13 @@ class MATHScenario(Scenario):
 
     def get_instances(self, output_path: str) -> List[Instance]:
         dataset = {}
-        data = typing.cast(DatasetDict, load_dataset("competition_math")).sort("problem").shuffle(seed=42)
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        data = (
+            typing.cast(DatasetDict, load_dataset("competition_math", cache_dir=cache_dir))
+            .sort("problem")
+            .shuffle(seed=42)
+        )
 
         def group_by_key(dataset_list, key):
             dataset_per_key = collections.defaultdict(list)

helm/benchmark/scenarios/medication_qa_scenario.py

@@ -0,0 +1,60 @@
+import os
+from typing import List
+
+import pandas as pd
+
+from helm.common.general import ensure_file_downloaded
+
+from .scenario import CORRECT_TAG, TEST_SPLIT, Input, Instance, Output, Reference, Scenario
+
+
+class MedicationQAScenario(Scenario):
+    """
+    The gold standard corpus for medication question answering introduced in the MedInfo 2019 paper
+    "Bridging the Gap between Consumers’ Medication Questions and Trusted Answers":
+    http://ebooks.iospress.nl/publication/51941
+
+    This dataset has consumer questions, as opposed to very clinical questions.
+
+    Paper citation:
+
+        @inproceedings{BenAbacha:MEDINFO19,
+        author    = {Asma {Ben Abacha} and Yassine Mrabet and Mark Sharp and
+                    Travis Goodwin and Sonya E. Shooshan and Dina Demner{-}Fushman},
+        title     = {Bridging the Gap between Consumers’ Medication Questions and Trusted Answers},
+        booktitle = {MEDINFO 2019},
+        year      = {2019},
+        }
+    """
+
+    SOURCE_REPO_URL = "https://github.com/abachaa/Medication_QA_MedInfo2019/raw/master/"
+    FILENAME = "MedInfo2019-QA-Medications.xlsx"
+
+    name = "medication_qa"
+    description = "MedInfo 2019 MedicationQA medication question answering task"
+    tags = ["knowledge", "generation", "question_answering", "biomedical"]
+
+    def download_medication_qa(self, path: str):
+        """download the .xlsx spreadsheet containing the question-answer pairs"""
+        ensure_file_downloaded(
+            source_url=os.path.join(self.SOURCE_REPO_URL, self.FILENAME),
+            target_path=os.path.join(path, self.FILENAME),
+            unpack=False,
+        )
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        self.download_medication_qa(output_path)
+        data_path = os.path.join(output_path, self.FILENAME)
+
+        data = pd.read_excel(data_path)
+        data = data[~data.Answer.isna()]  # remove rows missing answers
+        instances = [
+            Instance(
+                input=Input(row.Question),
+                references=[Reference(Output(row.Answer), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            for _, row in data.iterrows()
+        ]
+
+        return instances

helm/benchmark/scenarios/numeracy_scenario.py

@@ -358,7 +358,7 @@ def distance_paraboloid(point: List[int], rel_str: str, TOL: float = 1e-10):
         sols = []
         # Try each possible combined solution for x, y, z, λ
         for sol_xyz, val_λs in zip(sols_xyz, vals_λ):
-            val_λs = list(set(filter(lambda _: not _.is_symbol, val_λs)))  # get distinct values for λ if there are any
+            val_λs = tuple(set(filter(lambda _: not _.is_symbol, val_λs)))  # get distinct values for λ if there are any
             if len(val_λs) > 1:  # there can be at most one distinct value for λ
                 continue
             val_λ = val_λs[0] if val_λs else λ

helm/benchmark/scenarios/opinions_qa_scenario.py

@@ -108,7 +108,6 @@ class OpinionsQAScenario(Scenario):
         self.context: str = context
 
     def download_data(self, output_path: str):
-
         data_dir: str = os.path.join(output_path, "data")
         if not os.path.exists(data_dir):
             os.makedirs(data_dir)
@@ -150,14 +149,12 @@ class OpinionsQAScenario(Scenario):
             bios_df = pd.read_csv(bios_path, sep="\t")
 
         for split in all_splits:
-
             csv_path: str = csv_dict[split]
             assert os.path.exists(csv_path)
 
             question_df = self.read_survey_questions(csv_path)
 
             for qidx, (question, answers) in enumerate(zip(question_df["question"], question_df["options"])):
-
                 # Opinions QA test questions have no correct answer and thus we set it to be None by default
                 # for all test instances.
                 # In the case where context = steer-qa, we add demographic information in the form of a
@@ -182,7 +179,6 @@ class OpinionsQAScenario(Scenario):
                 else:
                     # context = "steer-bio"or "steer-portray"
                     for bio in bios_df["question"].values:
-
                         context = PassageQuestionInput(passage=bio, question=question + "\n")
                         instance = Instance(
                             context,

helm/benchmark/scenarios/scenario.py

@@ -25,6 +25,10 @@ DEFAULT_TEST_SIZE: int = 1000
 """ Reference tags """
 CORRECT_TAG: str = "correct"
 
+""" Asset tags (used for compiled outputs such as image2structure)"""
+ASSET_NAME_TAG: str = "asset_name"
+ASSET_PATH_TAG: str = "asset_path"
+
 # Reference tag functions for ranking scenarios.
 # @TODO: (For future) Should there be a base RankingScenario class?
 

helm/benchmark/scenarios/simple_scenarios.py

@@ -1,7 +1,128 @@
+"""Simple scenarios for debugging and for tutorials.
+
+NOTE: Typically, each scenario should be in its own file,
+but these scenarios are placed in the same module for
+tutorial purposes."""
+
 import random
 from typing import List
 
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class SimpleMCQAScenario(Scenario):
+    """Simple multiple-choice question answering scenario for tutorials and debugging.
+
+    The task is to answer questions about whether two-digit numbers are even or odd.
+
+    Example:
+
+        Answer the following questions with a single letter only.
+
+        Question: Is 24 even or odd?
+        A. Even
+        B. Odd
+        Answer: A"""
+
+    name = "simple_mcqa"
+    description = "Answer if two-digit numbers are even or odd."
+    tags = ["question answering"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        for i in range(10, 100):
+            # NOTE: For simplicity, the input text and reference output text
+            # is the same for all instances.
+            # However, for most question answering scenarios, the input text
+            # and reference output text can vary between questions.
+            input = Input(text=f"Is {i} even or odd?")
+            references = [
+                Reference(Output(text="Even"), tags=[CORRECT_TAG] if i % 2 == 0 else []),
+                Reference(Output(text="Odd"), tags=[CORRECT_TAG] if i % 2 == 1 else []),
+            ]
+            split = TRAIN_SPLIT if i <= 20 else TEST_SPLIT
+            instance = Instance(input=input, references=references, split=split)
+            instances.append(instance)
+        return instances
+
+
+class SimpleShortAnswerQAScenario(Scenario):
+    """Simple short answer question answering scenario for tutorials and debugging.
+
+    The task is to answer questions about whether two-digit numbers are even or odd.
+
+    Example:
+
+        Answer the following questions with a single word only.
+
+        Question: Is 24 even or odd?
+        Answer: Even"""
+
+    name = "simple_mcqa"
+    description = "Answer if two-digit numbers are even or odd."
+    tags = ["question answering"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        for i in range(10, 100):
+            # NOTE: For simplicity, the input text and reference output text
+            # is the same for all instances.
+            # However, for most question answering scenarios, the input text
+            # and reference output text can vary between questions.
+            input = Input(text=f"Is {i} even or odd?")
+            correct_answer = "Even" if i % 2 == 0 else "Odd"
+            # NOTE: Unlike multiple-choice question answering, only the correct
+            # references are needed for short-answer question answering.
+            references = [
+                Reference(Output(text=correct_answer), tags=[CORRECT_TAG]),
+            ]
+            split = TRAIN_SPLIT if i <= 20 else TEST_SPLIT
+            instance = Instance(input=input, references=references, split=split)
+            instances.append(instance)
+        return instances
+
+
+class SimpleClassificationScenario(Scenario):
+    """Simple multiple-choice question answering scenario for tutorials and debugging.
+
+    The task is to classify two-digit numbers as even or odd.
+
+    Example:
+
+        Classify the following numbers by their pairity. The classes are "Even" and "Odd".
+
+        Number: 24
+        Pairity: Even"""
+
+    name = "simple_classification"
+    description = "Classify numbers by pairity."
+    tags = ["classification"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        for i in range(10, 100):
+            input = Input(text=str(i))
+            # NOTE: For classification scenarios, the reference outputs should be the same
+            # for all instances, and should include both correct and incorrect classes.
+            # HELM only supports single-label classification. Exactly one reference
+            # should have the CORRECT_TAG tag.
+            references = [
+                Reference(Output(text="Even"), tags=[CORRECT_TAG] if i % 2 == 0 else []),
+                Reference(Output(text="Odd"), tags=[CORRECT_TAG] if i % 2 == 1 else []),
+            ]
+            split = TRAIN_SPLIT if i <= 20 else TEST_SPLIT
+            instance = Instance(input=input, references=references, split=split)
+            instances.append(instance)
+        return instances
 
 
 class Simple1Scenario(Scenario):

helm/benchmark/scenarios/test_math_scenario.py

@@ -1,9 +1,15 @@
+import pytest
 from tempfile import TemporaryDirectory
 
 from helm.benchmark.scenarios.math_scenario import MATHScenario
 from helm.benchmark.scenarios.scenario import Input, Output, Reference
 
 
+# TODO: Fix the test for newer versions of diffusers: https://github.com/stanford-crfm/helm/issues/2168
+@pytest.mark.skip(
+    reason="Incompatible with newer versions with diffusers>0.24.0. Fails with "
+    '"Loading a dataset cached in a LocalFileSystem is not supported"'
+)
 def test_math_scenario_get_instances():
     math_scenario = MATHScenario(subject="number_theory", level="1")
     with TemporaryDirectory() as tmpdir:

helm/benchmark/scenarios/test_scenario.py

@@ -1,10 +1,13 @@
-from helm.benchmark.run_specs import get_scenario_spec_tiny
-from helm.benchmark.scenarios.scenario import create_scenario, Scenario, Input, PassageQuestionInput
+from helm.benchmark.scenarios.scenario import ScenarioSpec, create_scenario, Scenario, Input, PassageQuestionInput
 
 
 class TestScenario:
     def setup_method(self, method):
-        self.scenario: Scenario = create_scenario(get_scenario_spec_tiny())
+        scenario_spec: ScenarioSpec = ScenarioSpec(
+            class_name="helm.benchmark.scenarios.simple_scenarios.Simple1Scenario",
+            args={"num_input_tokens": 5, "vocab_size": 20, "num_train_instances": 2, "num_test_instances": 2},
+        )
+        self.scenario: Scenario = create_scenario(scenario_spec)
 
     def test_render_lines(self):
         instances = self.scenario.get_instances(output_path="")

helm/benchmark/scenarios/test_simple_scenarios.py

@@ -0,0 +1,50 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.simple_scenarios import (
+    SimpleMCQAScenario,
+    SimpleShortAnswerQAScenario,
+    SimpleClassificationScenario,
+)
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_simple_mcqa_scenario():
+    scenario = SimpleMCQAScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 90
+    assert instances[0].input == Input(text="Is 10 even or odd?")
+    assert instances[0].references == [
+        Reference(output=Output(text="Even"), tags=[CORRECT_TAG]),
+        Reference(output=Output(text="Odd"), tags=[]),
+    ]
+    assert instances[0].split == "train"
+
+
+@pytest.mark.scenarios
+def test_simple_short_answer_qa_scenario():
+    scenario = SimpleShortAnswerQAScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 90
+    assert instances[0].input == Input(text="Is 10 even or odd?")
+    assert instances[0].references == [
+        Reference(output=Output(text="Even"), tags=[CORRECT_TAG]),
+    ]
+    assert instances[0].split == "train"
+
+
+@pytest.mark.scenarios
+def test_simple_classification_scenario():
+    scenario = SimpleClassificationScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 90
+    assert instances[0].input == Input(text="10")
+    assert instances[0].references == [
+        Reference(output=Output(text="Even"), tags=[CORRECT_TAG]),
+        Reference(output=Output(text="Odd"), tags=[]),
+    ]
+    assert instances[0].split == "train"

helm/benchmark/scenarios/thai_exam_scenario.py

@@ -0,0 +1,135 @@
+import os
+from typing import Dict, List
+import json
+
+from helm.common.general import ensure_file_downloaded
+from helm.common.hierarchical_logger import hlog
+from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+
+
+class ThaiExamScenario(Scenario):
+    """
+    ThaiExam, a benchmark comprising Thai multiple-choice examinations as follows:
+
+    ∙ ONET: The Ordinary National Educational Test (ONET) is an examination for students in Thailand.
+    We select the grade-12 ONET exam, which comprises 5 subjects and each question has 5 choices.
+    These subjects are Thai, English, Mathematics, Social Studies, and Science.
+    Amounting to a total of 170 questions and options.
+
+    ∙ IC: The Investment Consultant (IC) examination, a licensing test for investment professionals in Thailand.
+    Developed by the Stock Exchange of Thailand (SET), features 4 choices per question.
+    We extracted questions for levels 1, 2, and 3 resulting in a total of 95 questions and options.
+
+    ∙ TGAT: The Thai General Aptitude Test (TGAT), a national high school examination in Thailand.
+    Focuses on critical and logical thinking skills.
+    We collected a total of 90 questions and answers. The TGAT consists of four choices per question.
+
+    ∙ TPAT-1: The Thai Professional Aptitude Test 1 (TPAT-1) is a national high school examination in Thailand.
+    The Exam assesses students’ professional skills requirement in medical schools.
+    This subset contains reasoning and medical ethics. We collected a total of 116 questions and answers.
+    The TPAT-1 consists of 5 choices per question.
+
+    ∙ A-Level: An academic knowledge assessment examination (Applied Knowledge Level)
+    that covers general foundational subjects taught in schools.
+    The content assessed in this examination aligns with the curriculum guidelines
+    and emphasizes the practical application of knowledge in daily life.
+    We collected a total of 175 questions and answers.
+
+    We created and used these exams to evaluate the performance of the Typhoon models(https://arxiv.org/abs/2312.13951).
+
+    Prompt models using the following format
+
+        <input>                  # train
+        A. <reference>
+        B. <reference>
+        C. <reference>
+        D. <reference>
+        E. <reference>
+        Answer: <A/B/C/D/E>
+
+        x N (N-shot)
+
+        <input>                  # test
+        A. <reference1>
+        B. <reference2>
+        C. <reference3>
+        D. <reference4>
+        E. <reference5>
+        Answer:
+
+    For example:
+
+        ในระบบย่อยอาหารของมนุษย์ การดูดซึมสารอาหารส่วนใหญ่เกิดขึ้นที่อวัยวะใด?
+        A. ลำไส้เล็ก
+        B. ตับอ่อน
+        C. ลำไส้ใหญ่
+        D. กระเพาะอาหาร
+        E. หัวใจ
+        Answer: A
+
+        ข้อใดอธิบายเกี่ยวกับแรงไฟฟ้าได้ถูกต้อง?
+        A. เกิดได้โดยที่วัตถุไม่ต้องสัมผัสกัน
+        B. เป็นได้เฉพาะแรงผลักเท่านั้น
+        C. เป็นได้เฉพาะแรงดูดเท่านั้น
+        D. เป็นแรงต้านระหว่างวัตถุเท่านั้น
+        E. ถูกทุกข้อ
+        Answer:
+
+    Target: A
+    """
+
+    name = "thai_exam"
+    description = "ThaiExam benchmark comprising Thai multiple-choice examinations."
+    tags = ["knowledge", "multiple_choice"]
+
+    def __init__(self, exam: str):
+        super().__init__()
+        self.exam = exam
+
+    def download_thai_exam(self, path: str):
+        ensure_file_downloaded(
+            "https://storage.googleapis.com/thai_dataset/thai_exam.tar.gz",
+            target_path=path,
+            unpack=True,
+        )
+
+    def process_jsonl(self, jsonl_path: str, split: str) -> List[Instance]:
+        instances: List[Instance] = []
+        hlog(f"Reading {jsonl_path}")
+        with open(jsonl_path, "r") as f:
+            for line in f:
+                data = json.loads(line)
+                # for handle missing key incase of some subject doesn't have all 5 choices
+                answers = [data[key] for key in ["a", "b", "c", "d", "e"] if key in data and data[key] != ""]
+                answers_dict = dict(zip(["A", "B", "C", "D", "E"], answers))
+
+                question, correct_answer = data["question"], answers_dict[data["answer"].upper()]
+
+                def answer_to_reference(answer: str) -> Reference:
+                    return Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+
+                instance = Instance(
+                    input=Input(text=question),
+                    references=list(map(answer_to_reference, answers)),
+                    split=split,
+                )
+                instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path) -> List[Instance]:
+        data_path: str = os.path.join(output_path, "data")
+        self.download_thai_exam(data_path)
+
+        instances: List[Instance] = []
+        splits: Dict[str, str] = {
+            "train": TRAIN_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for split in splits:
+            jsonl_path: str = os.path.join(data_path, self.exam, f"{self.exam}_{split}.jsonl")
+            if not os.path.exists(jsonl_path):
+                hlog(f"{jsonl_path} doesn't exist, skipping")
+                continue
+            instances.extend(self.process_jsonl(jsonl_path, splits[split]))
+
+        return instances

helm/benchmark/scenarios/unitxt_scenario.py

@@ -0,0 +1,56 @@
+from typing import List
+from datasets import load_dataset
+
+from helm.benchmark.scenarios.scenario import (
+    Output,
+    Reference,
+    Scenario,
+    Instance,
+    Input,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    VALID_SPLIT,
+)
+
+
+class UnitxtScenario(Scenario):
+    """Integration with Unitxt: https://unitxt.rtfd.io/"""
+
+    name = "unitxt"
+    description = "Unitxt Scenarios"
+    tags = ["unitxt"]
+
+    UNITXT_SPLIT_NAME_TO_HELM_SPLIT_NAME = {
+        "train": TRAIN_SPLIT,
+        "test": TEST_SPLIT,
+        "validation": VALID_SPLIT,
+    }
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        self.kwargs = kwargs
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset_name = ",".join(f"{key}={value}" for key, value in self.kwargs.items())
+        dataset = load_dataset("unitxt/data", dataset_name, trust_remote_code=True)
+
+        instances: List[Instance] = []
+
+        for unitxt_split_name, helm_split_name in UnitxtScenario.UNITXT_SPLIT_NAME_TO_HELM_SPLIT_NAME.items():
+            for index, row in enumerate(dataset[unitxt_split_name]):
+                references = [
+                    Reference(
+                        output=Output(text=reference_text),
+                        tags=[CORRECT_TAG],
+                    )
+                    for reference_text in row["references"]
+                ]
+                instance = Instance(
+                    id=f"{unitxt_split_name}{index}",
+                    input=Input(text=row["source"]),
+                    references=references,
+                    split=helm_split_name,
+                )
+                instances.append(instance)
+        return instances

helm/benchmark/scenarios/verifiability_judgment_scenario.py

@@ -137,7 +137,9 @@ class VerifiabilityJudgementScenario(Scenario):
         for _, filesplit in split_to_filesplit.items():
             target_name = f"verifiability_judgments_{filesplit}.jsonl"
             target_path: str = os.path.join(data_path, target_name)
-            url: str = f"https://github.com/nelson-liu/evaluating-verifiability-in-generative-search-engines/raw/40bf37e3a4eca7d82515df2c800ec9605458d637/verifiability_judgments/{target_name}.gz"  # noqa: E501
+            url: str = (
+                f"https://github.com/nelson-liu/evaluating-verifiability-in-generative-search-engines/raw/40bf37e3a4eca7d82515df2c800ec9605458d637/verifiability_judgments/{target_name}.gz"  # noqa: E501
+            )
             ensure_file_downloaded(source_url=url, target_path=target_path)
             assert os.path.exists(target_path)
 

helm/benchmark/scenarios/vicuna_scenario.py

@@ -26,7 +26,7 @@ class VicunaScenario(Scenario):
             return self.category == "all" or raw["category"] == self.category
 
         # Download the raw data
-        source_url = "https://raw.githubusercontent.com/lm-sys/FastChat/main/fastchat/eval/table/question.jsonl"
+        source_url = "https://raw.githubusercontent.com/lm-sys/FastChat/v0.2.5/fastchat/eval/table/question.jsonl"
         data_path: str = os.path.join(output_path, "vicuna_questions.jsonl")
 
         ensure_file_downloaded(