crfm-helm 0.5.2__py3-none-any.whl → 0.5.4__py3-none-any.whl

helm/benchmark/scenarios/call_center_scenario.py

@@ -0,0 +1,84 @@
+import datasets
+import os
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Output,
+    Reference,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    Input,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class CallCenterSummarizationScenario(Scenario):
+    """Call center summarization."""
+
+    name = "call_center_summarization"
+    description = "Call center summarization."
+    tags = ["call_center"]
+
+    def __init__(self, subset: str):
+        super().__init__()
+        self.subset = subset
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset("yifanmai/call-center", self.subset, split="test", cache_dir=cache_dir)
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=row["transcript"])
+            instance = Instance(input=input, references=[], split=TEST_SPLIT)
+            instances.append(instance)
+        return instances
+
+
+class CallCenterSummarizationPairwiseComparisonScenario(Scenario):
+    """Call center summarization."""
+
+    name = "call_center_summarization_pairwise_comparison"
+    description = "Call center summarization."
+    tags = ["call_center"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "yifanmai/call-center", "summarization_with_annotations", split="test", cache_dir=cache_dir
+        )
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=row["transcript"])
+            reference = Reference(output=Output(text=row["gpt-4o-mini-2024-07-18_summary"]), tags=[CORRECT_TAG])
+            instance = Instance(input=input, references=[reference], split=TEST_SPLIT)
+            instances.append(instance)
+        return instances
+
+
+class CallCenterSummarizationKeyPointsRecallScenario(Scenario):
+    """Call center summarization."""
+
+    name = "call_center_summarization_key_points_recall"
+    description = "Call center summarization."
+    tags = ["call_center"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "yifanmai/call-center", "summarization_with_annotations", split="test", cache_dir=cache_dir
+        )
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=row["transcript"])
+            references = [
+                Reference(output=Output(text=key_point), tags=[CORRECT_TAG])
+                for key_point in row["gpt-4o-mini-2024-07-18_key_points"]
+            ]
+            instance = Instance(input=input, references=references, split=TEST_SPLIT)
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/decodingtrust_stereotype_bias_scenario.py

@@ -1,7 +1,7 @@
 import json
 import os
 from typing import List, Dict
-from .scenario import Scenario, Instance, Input
+from .scenario import Scenario, Instance, Input, TEST_SPLIT
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import Reference, Output
 
@@ -61,6 +61,7 @@ class DecodingTrustStereotypeBiasScenario(Scenario):
                             tags=[stereotype_topic_tag, demographic_group_tag, sys_prompt_type_tag],
                         )
                     ],
+                    split=TEST_SPLIT,
                 )
                 instances.append(instance)
 

helm/benchmark/scenarios/ewok_scenario.py

@@ -0,0 +1,116 @@
+import datasets
+import os
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    Input,
+    Output,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class EWoKScenario(Scenario):
+    """Elements of World Knowledge (EWoK)
+
+    Elements of World Knowledge (EWoK) is a framework for evaluating world modeling in
+    language models by testing their ability to use knowledge of a concept to match a
+    target text with a plausible/implausible context. EWoK targets specific concepts
+    from multiple knowledge domains known to be vital for world modeling in humans.
+    Domains range from social interactions (help/hinder) to spatial relations (left/right).
+    Both, contexts and targets are minimal pairs. Objects, agents, and locations in the items
+    can be flexibly filled in enabling easy generation of multiple controlled datasets.
+
+    EWoK-CORE-1.0 is a dataset of 4,374 items covering 11 world knowledge domains."""
+
+    name = "ewok"
+    description = (
+        "Elements of World Knowledge (EWoK) is a benchmark for evaluating world modeling by testing their ability to "
+        "use knowledge of a concept to match a target text with a plausible/implausible context."
+    )
+    tags = ["world knowledge"]
+
+    DOMAINS = [
+        "agent_properties",
+        "material_dynamics",
+        "material_properties",
+        "physical_dynamics",
+        "physical_interactions",
+        "physical_relations",
+        "quantitative_properties",
+        "social_interactions",
+        "social_properties",
+        "social_relations",
+        "spatial_relations",
+    ]
+
+    def __init__(self, domain: str = "all"):
+        super().__init__()
+        if domain != "all" and domain not in EWoKScenario.DOMAINS:
+            raise Exception(f"Unknown domain '{domain}', valid domains are {EWoKScenario.DOMAINS}")
+        self.domain = domain.replace("_", "-")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        next_instance_index = 0
+
+        def generate_instance_id() -> str:
+            nonlocal next_instance_index
+            instance_id = f"id{next_instance_index}"
+            next_instance_index += 1
+            return instance_id
+
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+
+        # TODO: Switch this to the production dataset when available.
+        dataset = datasets.load_dataset(
+            "ewok-core/ewok-core-1.0",
+            split="test",
+            revision="34d912a608066c92e2990a0328ffc3bd9a716042",
+            cache_dir=cache_dir,
+        )
+
+        instances: List[Instance] = []
+        for row in dataset:
+            contexts = [row["Context1"], row["Context2"]]
+            targets = [row["Target1"], row["Target2"]]
+            # References are category ID, followed by level 2, 3 and 4 category names.
+            for target_index, target in enumerate(targets):
+                input = Input(text=target)
+                references = [
+                    Reference(output=Output(text=context), tags=[CORRECT_TAG] if context_index == target_index else [])
+                    for context_index, context in enumerate(contexts)
+                ]
+                instance = Instance(id=generate_instance_id(), input=input, references=references, split=TEST_SPLIT)
+                # Filtering by domain after generate instance IDs,
+                # so that instance IDs for an item is invariant regardless of domain filtering.
+                if self.domain == "all" or self.domain == row["Domain"]:
+                    instances.append(instance)
+        instances.extend(
+            [
+                Instance(
+                    id=generate_instance_id(),
+                    input=Input(text="I drew a ball from the bag."),
+                    references=[
+                        Reference(output=Output(text="The bag is full of blocks."), tags=[]),
+                        Reference(output=Output(text="The bag is full of balls."), tags=[CORRECT_TAG]),
+                    ],
+                    split=TRAIN_SPLIT,
+                ),
+                Instance(
+                    id=generate_instance_id(),
+                    input=Input(text="The boy chose to eat a cookie."),
+                    references=[
+                        Reference(output=Output(text="The boy likes cookies."), tags=[CORRECT_TAG]),
+                        Reference(output=Output(text="The boy does not like cookies."), tags=[]),
+                    ],
+                    split=TRAIN_SPLIT,
+                ),
+            ]
+        )
+        return instances

helm/benchmark/scenarios/fin_qa_scenario.py

@@ -18,6 +18,8 @@ from helm.benchmark.scenarios.scenario import (
 DATASET_URL_PREFIX = "https://github.com/czyssrs/FinQA/raw/0f16e2867befa6840783e58be38c9efb9229d742/dataset/"
 INSTRUCTIONS = """Presented with a financial report consisting of textual contents and a structured table, given a question, generate the reasoning program in the domain specific langauge (DSL) that will be executed to get the answer.
 
+Respond with only the program in the DSL for the last question, without any preamble, elaboration, or working steps. Do not respond with anything that is not part of the DSL.
+
 The DSL consists of mathematical operations and table operations as executable programs. The program consists of a sequence of operations. Each operation takes a list of arguments.
 
 There are 6 mathematical operations: add, subtract, multiply, divide, greater, exp, and 4 table aggregation operations table-max, table-min, table-sum, table-average, that apply aggregation operations on table rows. The mathematical operations take arguments of either numbers from the given reports, or a numerical result from a previous step.

helm/benchmark/scenarios/financebench_scenario.py

@@ -0,0 +1,53 @@
+import dataclasses
+import json
+import os
+import random
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    Input,
+    Output,
+)
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class FinanceBenchScenario(Scenario):
+    """FinanceBench"""
+
+    name = "financebench"
+    description = "FinanceBench"
+    tags = ["finance"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        target_path = os.path.join(cache_dir, "financebench_open_source.jsonl")
+        url: str = (
+            "https://raw.githubusercontent.com/patronus-ai/financebench/d7beebe5e739e0b806ab4443c1b3e23f51804acf/data/financebench_open_source.jsonl"  # noqa: E501
+        )
+        ensure_file_downloaded(source_url=url, target_path=target_path)
+
+        instances: List[Instance] = []
+        with open(target_path) as f:
+            for line in f:
+                row = json.loads(line)
+                instance_id = row["financebench_id"]
+                question = row["question"]
+                answer = row["answer"]
+                evidence = row["evidence"][0]["evidence_text_full_page"]
+                input_text = f"Evidence: {evidence}\nQuestion: {question}"
+                input = Input(text=input_text)
+                references = [Reference(output=Output(text=answer), tags=[CORRECT_TAG])]
+                instance = Instance(id=instance_id, input=input, references=references, split=TEST_SPLIT)
+                instances.append(instance)
+        random.seed(0)
+        train_indexes = random.sample(list(range(len(instances))), k=10)
+        for train_index in train_indexes:
+            instances[train_index] = dataclasses.replace(instances[train_index], split=TRAIN_SPLIT)
+        return instances

helm/benchmark/scenarios/harm_bench_scenario.py

@@ -0,0 +1,59 @@
+import os
+import pandas as pd
+from typing import List
+
+from helm.common.general import ensure_file_downloaded
+
+from .scenario import Scenario, Instance, Input, TEST_SPLIT, Reference, Output
+
+
+class HarmBenchScenario(Scenario):
+    """
+    HarmBench is a standardized evaluation framework for automated red teaming.
+    HarmBench identifies key considerations previously unaccounted for in red teaming
+    evaluations and systematically designed prompts that meet these criteria.
+
+    https://arxiv.org/abs/2402.04249
+    """
+
+    name = "harm_bench"
+    description = "HarmBench dataset - standard and contextual attacks"
+    tags = ["safety"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = os.path.join(output_path, "data")
+        os.makedirs(data_path, exist_ok=True)
+        url_prefix = "https://raw.githubusercontent.com/centerforaisafety/HarmBench/8e1604d1171fe8a48d8febecd22f600e462bdcdd/data/"  # noqa: E501
+        outf_path = os.path.join(data_path, "harmbench_behaviors_text_all.csv")
+        ensure_file_downloaded(
+            source_url=url_prefix + "behavior_datasets/harmbench_behaviors_text_all.csv",
+            target_path=outf_path,
+            unpack=False,
+        )
+
+        df = pd.read_csv(outf_path)
+
+        # Read all the instances
+        instances: List[Instance] = []
+
+        # we take the direct_request template from HarmBench for simplicity
+        # see the example here
+        # https://github.com/centerforaisafety/HarmBench/blob/main/baselines/direct_request/direct_request.py
+        #
+        # in the future we can use other types of templates including various red team attacks
+        for i, row in df.iterrows():
+            if row["FunctionalCategory"] == "contextual":
+                context = row["ContextString"]
+                input_text = context + "\n\n---\n\n" + row["Behavior"]
+            else:
+                input_text = row["Behavior"]
+            input = Input(text=input_text)
+            tag = row["Tags"]
+            id = row["BehaviorID"]
+            references = [Reference(output=Output(text=row["SemanticCategory"]), tags=[])]
+            if tag != "":
+                instance = Instance(input=input, split=TEST_SPLIT, references=references, id=id)
+            else:
+                instance = Instance(input=input, split=TEST_SPLIT, references=references, sub_split=tag, id=id)
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/raft_scenario.py

@@ -40,7 +40,7 @@ def get_raft_prompt_settings(subset: str, cache_dir: str):
     return field_ordering[subset], instructions[subset]
 
 
-def get_raft_instructions(subset: str, cache_dir: str):
+def get_raft_instructions(subset: str, cache_dir: str) -> str:
     return get_raft_prompt_settings(subset, cache_dir)[1]
 
 

helm/benchmark/scenarios/scenario.py

@@ -25,7 +25,7 @@ DEFAULT_TEST_SIZE: int = 1000
 """ Reference tags """
 CORRECT_TAG: str = "correct"
 
-""" Asset tags (used for compiled outputs such as image2structure)"""
+""" Asset tags (used for compiled outputs such as image2struct)"""
 ASSET_NAME_TAG: str = "asset_name"
 ASSET_PATH_TAG: str = "asset_path"
 

helm/benchmark/scenarios/simple_safety_tests_scenario.py

@@ -0,0 +1,33 @@
+from typing import List
+from datasets import load_dataset
+
+from .scenario import Scenario, Instance, Input, TEST_SPLIT, Reference, Output
+
+
+class SimpleSafetyTestsScenario(Scenario):
+    """
+    The test suite comprises 100 test prompts across five harm areas that LLMs,
+    for the vast majority of applications, should refuse to comply with
+    https://huggingface.co/datasets/Bertievidgen/SimpleSafetyTests
+    """
+
+    name = "SimpleSafetyTests"
+    description = "SimpleSafetyTests dataset"
+    tags = ["safety"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # read explicit data
+        dataset = load_dataset(
+            "Bertievidgen/SimpleSafetyTests", split="test", revision="98223c5d8c4059c8f4d8fe2fec8720ee8a20d3c5"
+        )
+
+        # Read all the instances
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=row["prompt"])
+            references = [
+                Reference(output=Output(text=row[column_name]), tags=[]) for column_name in ["category", "harm_area"]
+            ]
+            instance = Instance(input=input, references=references, split=TEST_SPLIT)
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/test_commonsense_scenario.py

@@ -0,0 +1,21 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.commonsense_scenario import OpenBookQA
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_openbookqa_scenario():
+    scenario = OpenBookQA()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 5457
+    assert instances[0].input == Input(text="The sun is responsible for")
+    assert instances[0].references == [
+        Reference(output=Output(text="puppies learning new tricks"), tags=[]),
+        Reference(output=Output(text="children growing up and getting old"), tags=[]),
+        Reference(output=Output(text="flowers wilting in a vase"), tags=[]),
+        Reference(output=Output(text="plants sprouting, blooming and wilting"), tags=[CORRECT_TAG]),
+    ]
+    assert instances[0].split == "train"

helm/benchmark/scenarios/test_ewok_scenario.py

@@ -0,0 +1,25 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from datasets.exceptions import DatasetNotFoundError
+
+from helm.benchmark.scenarios.ewok_scenario import EWoKScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+@pytest.mark.scenarios
+def test_ewok_scenario():
+    scenario = EWoKScenario()
+    with TemporaryDirectory() as tmpdir:
+        try:
+            instances = scenario.get_instances(tmpdir)
+        except DatasetNotFoundError:
+            pytest.skip("Unable to access gated dataset on Hugging Face Hub; skipping test")
+    assert len(instances) == 8748
+    assert "believes" in instances[0].input.text
+    assert len(instances[0].references) == 2
+    assert "inside" in instances[0].references[0].output.text
+    assert instances[0].references[0].tags == [CORRECT_TAG]
+    assert "outside" in instances[0].references[1].output.text
+    assert instances[0].references[1].tags == []
+    assert instances[0].split == "test"

helm/benchmark/scenarios/test_financebench_scenario.py

@@ -0,0 +1,26 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.financebench_scenario import FinanceBenchScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, TEST_SPLIT, TRAIN_SPLIT
+
+
+@pytest.mark.scenarios
+def test_air_2024_scenario_get_instances():
+    scenario = FinanceBenchScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 150
+    assert len([instance for instance in instances if instance.split == TRAIN_SPLIT]) == 10
+    assert (
+        "Evidence: Table of Contents \n3M Company and Subsidiaries\nConsolidated Statement of Cash Flow s\n"  # noqa: E501
+        in instances[0].input.text
+    )
+    assert (
+        "Question: What is the FY2018 capital expenditure amount (in USD millions) for 3M? Give a response to the question by relying on the details shown in the cash flow statement."  # noqa: E501
+        in instances[0].input.text
+    )
+    assert len(instances[0].references) == 1
+    assert instances[0].references[0].output.text == "$1577.00"
+    assert instances[0].references[0].tags == [CORRECT_TAG]
+    assert instances[0].split == TEST_SPLIT

helm/benchmark/scenarios/test_gsm_scenario.py

@@ -0,0 +1,31 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.gsm_scenario import GSM8KScenario
+from helm.benchmark.scenarios.scenario import Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_gsm_scenario_get_instances():
+    math_scenario = GSM8KScenario()
+    with TemporaryDirectory() as tmpdir:
+        actual_instances = math_scenario.get_instances(tmpdir)
+    assert len(actual_instances) == 8792
+    assert actual_instances[0].input == Input(
+        text=(
+            "Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many"
+            " clips did Natalia sell altogether in April and May?"
+        )
+    )
+    assert actual_instances[0].references == [
+        Reference(
+            output=Output(
+                text=(
+                    "Natalia sold 48/2 = <<48/2=24>>24 clips in May. Natalia sold 48+24 = <<48+24=72>>72 clips"
+                    " altogether in April and May. The answer is 72."
+                )
+            ),
+            tags=["correct"],
+        )
+    ]
+    assert actual_instances[0].split == "train"

helm/benchmark/scenarios/test_legalbench_scenario.py

@@ -0,0 +1,30 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.legalbench_scenario import LegalBenchScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_legalbench_scenario():
+    scenario = LegalBenchScenario(subset="abercrombie")
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 100
+    assert instances[0].input == Input(text='Description: The mark "Ivory" for a product made of elephant tusks.')
+    assert instances[0].references == [
+        Reference(output=Output(text="generic"), tags=["correct"]),
+    ]
+    assert instances[0].split == "train"
+
+    scenario = LegalBenchScenario(subset="proa")
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 100
+    assert instances[0].input == Input(
+        text="Statute: Amendments to pleadings must be filed within 15 days of the filing of the initial pleading."
+    )
+    assert instances[0].references == [
+        Reference(output=Output(text="No"), tags=[CORRECT_TAG]),
+    ]
+    assert instances[0].split == "train"

helm/benchmark/scenarios/test_math_scenario.py

@@ -5,18 +5,12 @@ from helm.benchmark.scenarios.math_scenario import MATHScenario
 from helm.benchmark.scenarios.scenario import Input, Output, Reference
 
 
-# TODO: Fix the test for newer versions of diffusers: https://github.com/stanford-crfm/helm/issues/2168
-@pytest.mark.skip(
-    reason="Incompatible with newer versions with diffusers>0.24.0. Fails with "
-    '"Loading a dataset cached in a LocalFileSystem is not supported"'
-)
+@pytest.mark.scenarios
 def test_math_scenario_get_instances():
     math_scenario = MATHScenario(subject="number_theory", level="1")
     with TemporaryDirectory() as tmpdir:
         actual_instances = math_scenario.get_instances(tmpdir)
     assert len(actual_instances) == 77
     assert actual_instances[0].input == Input(text="What is the remainder when (99)(101) is divided by 9?")
-    assert actual_instances[0].references == [
-        Reference(output=Output(text="0", multimedia_content=None), tags=["correct"])
-    ]
+    assert actual_instances[0].references == [Reference(output=Output(text="0"), tags=["correct"])]
     assert actual_instances[0].split == "train"

helm/benchmark/scenarios/test_med_qa_scenario.py

@@ -0,0 +1,30 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.med_qa_scenario import MedQAScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_med_qa_scenario():
+    scenario = MedQAScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 12723
+    assert instances[0].input == Input(
+        text=(
+            "A 23-year-old pregnant woman at 22 weeks gestation presents with burning upon urination. She states it"
+            " started 1 day ago and has been worsening despite drinking more water and taking cranberry extract. She"
+            " otherwise feels well and is followed by a doctor for her pregnancy. Her temperature is 97.7°F (36.5°C),"
+            " blood pressure is 122/77 mmHg, pulse is 80/min, respirations are 19/min, and oxygen saturation is 98% on"
+            " room air. Physical exam is notable for an absence of costovertebral angle tenderness and a gravid uterus."
+            " Which of the following is the best treatment for this patient?"
+        )
+    )
+    assert instances[0].references == [
+        Reference(output=Output(text="Ampicillin"), tags=[]),
+        Reference(output=Output(text="Ceftriaxone"), tags=[]),
+        Reference(output=Output(text="Doxycycline"), tags=[]),
+        Reference(output=Output(text="Nitrofurantoin"), tags=[CORRECT_TAG]),
+    ]
+    assert instances[0].split == "train"

helm/benchmark/scenarios/test_mmlu_scenario.py

@@ -0,0 +1,33 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.mmlu_scenario import MMLUScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_mmlu_scenario():
+    with TemporaryDirectory() as tmpdir:
+        scenario = MMLUScenario(subject="abstract_algebra")
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 116
+        assert instances[0].input == Input(text="Find all c in Z_3 such that Z_3[x]/(x^2 + c) is a field.")
+        assert instances[0].references == [
+            Reference(output=Output(text="0"), tags=[]),
+            Reference(output=Output(text="1"), tags=[CORRECT_TAG]),
+            Reference(output=Output(text="2"), tags=[]),
+            Reference(output=Output(text="3"), tags=[]),
+        ]
+        assert instances[0].split == "train"
+
+        scenario = MMLUScenario(subject="anatomy")
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 154
+        assert instances[0].input == Input(text="What is the embryological origin of the hyoid bone?")
+        assert instances[0].references == [
+            Reference(output=Output(text="The first pharyngeal arch"), tags=[]),
+            Reference(output=Output(text="The first and second pharyngeal arches"), tags=[]),
+            Reference(output=Output(text="The second pharyngeal arch"), tags=[]),
+            Reference(output=Output(text="The second and third pharyngeal arches"), tags=[CORRECT_TAG]),
+        ]
+        assert instances[0].split == "train"