crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/test_aratrust_scenario.py

@@ -0,0 +1,21 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.aratrust_scenario import AraTrustScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input
+
+
+@pytest.mark.scenarios
+def test_aratrust_get_instances():
+    scenario = AraTrustScenario(category="all")
+    with TemporaryDirectory() as tmpdir:
+        actual_instances = scenario.get_instances(tmpdir)
+    assert len(actual_instances) == 522
+    assert actual_instances[0].id == "id0"
+    assert actual_instances[0].input == Input(
+        text=("حيوان لا يبيض ولا يلد ما هو؟\n\nأ) أنثى الحيوان\n ب) ذكر الحيوان\n ج) كل ما سبق\n")
+    )
+    assert len(actual_instances[0].references) == 1
+    assert actual_instances[0].references[0].output.text == "ب"
+    assert actual_instances[0].references[0].tags == [CORRECT_TAG]
+    assert actual_instances[0].split == "test"

helm/benchmark/scenarios/test_bigcodebench_scenario.py

@@ -0,0 +1,26 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.bigcodebench_scenario import BigCodeBenchScenario
+from helm.benchmark.scenarios.scenario import Input, TEST_SPLIT
+
+
+@pytest.mark.scenarios
+def test_bigcodebench_scenario_get_instances():
+    bigcodebench_scenario = BigCodeBenchScenario("v0.1.2")
+    with TemporaryDirectory() as tmpdir:
+        instances = bigcodebench_scenario.get_instances(tmpdir)
+    assert len(instances) == 1140
+    assert instances[0].id == "BigCodeBench/0"
+    assert instances[0].input == Input(
+        text=(
+            "Calculates the average of the sums of absolute differences between each pair "
+            "of consecutive numbers for all permutations of a given list. Each permutation "
+            "is shuffled before calculating the differences. Args: - numbers (list): A list "
+            "of numbers. Default is numbers from 1 to 10.\nThe function should output with:\n"
+            "    float: The average of the sums of absolute differences for each shuffled permutation "
+            "of the list.\nYou should write self-contained code starting with:\n```\nimport itertools\n"
+            "from random import shuffle\ndef task_func(numbers=list(range(1, 3))):\n```"
+        )
+    )
+    assert instances[0].split == TEST_SPLIT

helm/benchmark/scenarios/test_bluex_scenario.py

@@ -0,0 +1,59 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.bluex_scenario import BLUEXScenario
+from helm.benchmark.scenarios.scenario import TEST_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_bluex_scenario():
+    scenario = BLUEXScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+
+    assert len(instances) > 100
+
+    assert instances[100].split == TEST_SPLIT
+
+    assert instances[0].input.text.startswith("Rubião fitava a enseada, - eram oito horas da manhã Quem o visse")
+
+    assert len(instances[0].input.text) == 1011
+
+    assert instances[0].references == [
+        Reference(
+            output=Output(
+                text='a contemplação das paisagens naturais, como se lê em "ele admirava aquele pedaço de água quieta".'
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text='a presença de um narrador-personagem, como se lê em "em verdade vos digo que pensava em '
+                'outra coisa".'
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text='a sobriedade do protagonista ao avaliar o seu percurso, como se lê em "Cotejava o passado com '
+                "o presente."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text='o sentido místico e fatalista que rege os destinos, como se lê em "Deus escreve direito por '
+                'linhas tortas".'
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text='a reversibilidade entre o cômico e o trágico, como se lê em "de modo que o que parecia uma '
+                'desgraça...".'
+            ),
+            tags=[CORRECT_TAG],
+        ),
+    ]
+
+    assert instances[0].references[4].is_correct

helm/benchmark/scenarios/test_commonsense_scenario.py

@@ -0,0 +1,21 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.commonsense_scenario import OpenBookQA
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_openbookqa_scenario():
+    scenario = OpenBookQA()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 5457
+    assert instances[0].input == Input(text="The sun is responsible for")
+    assert instances[0].references == [
+        Reference(output=Output(text="puppies learning new tricks"), tags=[]),
+        Reference(output=Output(text="children growing up and getting old"), tags=[]),
+        Reference(output=Output(text="flowers wilting in a vase"), tags=[]),
+        Reference(output=Output(text="plants sprouting, blooming and wilting"), tags=[CORRECT_TAG]),
+    ]
+    assert instances[0].split == "train"

helm/benchmark/scenarios/test_czech_bank_qa_scenario.py

@@ -0,0 +1,18 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.czech_bank_qa_scenario import CzechBankQAScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input
+
+
+@pytest.mark.scenarios
+def test_czech_bank_qa_scenario_get_instances():
+    scenario = CzechBankQAScenario(config_name="default")
+    with TemporaryDirectory() as tmpdir:
+        actual_instances = scenario.get_instances(tmpdir)
+    assert len(actual_instances) == 30
+    assert actual_instances[0].input == Input(text="Get the total number of accounts in the system")
+    assert len(actual_instances[0].references) == 1
+    assert actual_instances[0].references[0].tags == [CORRECT_TAG]
+    assert actual_instances[0].references[0].output.text == "SELECT COUNT(*) FROM ACCOUNT"
+    assert actual_instances[0].split == "test"

helm/benchmark/scenarios/test_enem_challenge_scenario.py

@@ -0,0 +1,53 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.enem_challenge_scenario import ENEMChallengeScenario
+from helm.benchmark.scenarios.scenario import TEST_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_enem_challenge_scenario():
+    enem_scenario = ENEMChallengeScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = enem_scenario.get_instances(tmpdir)
+    assert len(instances) == 1431
+    assert instances[0].split == TEST_SPLIT
+
+    assert instances[0].input.text.startswith(
+        "A atmosfera terrestre é composta pelos gases nitrogênio (N2) e oxigênio (O2)"
+    )
+    assert len(instances[0].input.text) == 1163
+
+    assert instances[0].references == [
+        Reference(
+            output=Output(
+                text="reduzir o calor irradiado pela Terra mediante a substituição da produção primária pela industrialização refrigerada. "  # noqa: E501
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="promover a queima da biomassa vegetal, responsável pelo aumento do efeito estufa devido à produção de CH4. "  # noqa: E501
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="reduzir o desmatamento, mantendo-se, assim, o potencial da vegetação em absorver o CO2 da atmosfera. "  # noqa: E501
+            ),
+            tags=[CORRECT_TAG],
+        ),
+        Reference(
+            output=Output(
+                text="aumentar a concentração atmosférica de H2O, molécula capaz de absorver grande quantidade de calor. "  # noqa: E501
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="remover moléculas orgânicas polares da atmosfera, diminuindo a capacidade delas de reter calor. "  # noqa: E501
+            ),
+            tags=[],
+        ),
+    ]
+    assert instances[0].references[2].is_correct

helm/benchmark/scenarios/test_ewok_scenario.py

@@ -0,0 +1,29 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from datasets.exceptions import DatasetNotFoundError
+
+from helm.benchmark.scenarios.ewok_scenario import EWoKScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+@pytest.mark.scenarios
+def test_ewok_scenario():
+    scenario = EWoKScenario()
+    with TemporaryDirectory() as tmpdir:
+        try:
+            instances = scenario.get_instances(tmpdir)
+        except (DatasetNotFoundError, Exception):
+            # Catch generic Exception here because the Hugging Face Hub client can raise
+            # an authentication issue as the following exception:
+            # huggingface_hub.utils._errors.GatedRepoError: 401 Client Error.
+            # but GatedRepoError is a type in a private package, so we avoid referencing it
+            pytest.skip("Unable to access gated dataset on Hugging Face Hub; skipping test")
+    assert len(instances) == 8750
+    assert "believes" in instances[0].input.text
+    assert len(instances[0].references) == 2
+    assert "inside" in instances[0].references[0].output.text
+    assert instances[0].references[0].tags == [CORRECT_TAG]
+    assert "outside" in instances[0].references[1].output.text
+    assert instances[0].references[1].tags == []
+    assert instances[0].split == "test"

helm/benchmark/scenarios/test_exams_multilingual_scenario.py

@@ -0,0 +1,29 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.exams_multilingual_scenario import EXAMSMultilingualScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, TRAIN_SPLIT, Input
+
+
+@pytest.mark.scenarios
+def test_exam_multilingual_scenario_get_instances():
+    scenario = EXAMSMultilingualScenario(language="Bulgarian", subject="Physics")
+    with TemporaryDirectory() as tmpdir:
+        actual_instances = scenario.get_instances(tmpdir)
+    assert len(actual_instances) == 393
+    assert actual_instances[0].id == "4c05bbb8-7729-11ea-9116-54bef70b159e"
+    assert actual_instances[0].input == Input(text="Наелектризирането по индукция се обяснява с: ")
+    assert len(actual_instances[0].references) == 4
+    assert actual_instances[0].references[0].output.text == "преразпределение на положителните йони в тялото"
+    assert actual_instances[0].references[0].tags == []
+    assert (
+        actual_instances[0].references[1].output.text == "предаване на електрони от неутрално на наелектризирано тяло"
+    )
+    assert actual_instances[0].references[1].tags == []
+    assert (
+        actual_instances[0].references[2].output.text == "предаване на електрони от наелектризирано на неутрално тяло"
+    )
+    assert actual_instances[0].references[2].tags == []
+    assert actual_instances[0].references[3].output.text == "преразпределение на свободните електрони в тялото"
+    assert actual_instances[0].references[3].tags == [CORRECT_TAG]
+    assert actual_instances[0].split == TRAIN_SPLIT

helm/benchmark/scenarios/test_financebench_scenario.py

@@ -0,0 +1,26 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.financebench_scenario import FinanceBenchScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, TEST_SPLIT, TRAIN_SPLIT
+
+
+@pytest.mark.scenarios
+def test_air_2024_scenario_get_instances():
+    scenario = FinanceBenchScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 150
+    assert len([instance for instance in instances if instance.split == TRAIN_SPLIT]) == 10
+    assert (
+        "Evidence: Table of Contents \n3M Company and Subsidiaries\nConsolidated Statement of Cash Flow s\n"  # noqa: E501
+        in instances[0].input.text
+    )
+    assert (
+        "Question: What is the FY2018 capital expenditure amount (in USD millions) for 3M? Give a response to the question by relying on the details shown in the cash flow statement."  # noqa: E501
+        in instances[0].input.text
+    )
+    assert len(instances[0].references) == 1
+    assert instances[0].references[0].output.text == "$1577.00"
+    assert instances[0].references[0].tags == [CORRECT_TAG]
+    assert instances[0].split == TEST_SPLIT

helm/benchmark/scenarios/test_gold_commodity_news_scenario.py

@@ -0,0 +1,18 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.gold_commodity_news_scenario import GoldCommodityNewsScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_legalbench_scenario():
+    scenario = GoldCommodityNewsScenario(category="price_or_not")
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 11412
+    assert instances[0].input == Input(text="april gold down 20 cents to settle at $1,116.10/oz")
+    assert instances[0].references == [
+        Reference(output=Output(text="Yes"), tags=[CORRECT_TAG]),
+    ]
+    assert instances[0].split == "test"

helm/benchmark/scenarios/test_gpqa_scenario.py

@@ -0,0 +1,44 @@
+import pytest
+from tempfile import TemporaryDirectory
+from helm.benchmark.scenarios.gpqa_scenario import GPQAScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+@pytest.mark.scenarios
+def test_gpqa_scenario():
+    with TemporaryDirectory() as tmpdir:
+        scenario = GPQAScenario(subset="gpqa_main")
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 448
+        assert instances[0].split == "test"
+        assert len(instances[0].input.text) == 689
+        references = instances[0].references
+        assert len(references[0].output.text) == 10
+        assert len(references[1].output.text) == 6
+        assert len(references[2].output.text) == 9
+        assert len(references[3].output.text) == 7
+        assert references[3].tags == [CORRECT_TAG]
+
+        scenario = GPQAScenario(subset="gpqa_diamond")
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 198
+        assert instances[0].split == "test"
+        assert len(instances[0].input.text) == 262
+        references = instances[0].references
+        assert len(references[0].output.text) == 8
+        assert len(references[1].output.text) == 9
+        assert len(references[2].output.text) == 8
+        assert len(references[3].output.text) == 8
+        assert references[3].tags == [CORRECT_TAG]
+
+        scenario = GPQAScenario(subset="gpqa_extended")
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 546
+        assert instances[0].split == "test"
+        assert len(instances[0].input.text) == 689
+        references = instances[0].references
+        assert len(references[0].output.text) == 10
+        assert len(references[1].output.text) == 6
+        assert len(references[2].output.text) == 9
+        assert len(references[3].output.text) == 7
+        assert references[3].tags == [CORRECT_TAG]

helm/benchmark/scenarios/test_gsm_scenario.py

@@ -0,0 +1,31 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.gsm_scenario import GSM8KScenario
+from helm.benchmark.scenarios.scenario import Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_gsm_scenario_get_instances():
+    math_scenario = GSM8KScenario()
+    with TemporaryDirectory() as tmpdir:
+        actual_instances = math_scenario.get_instances(tmpdir)
+    assert len(actual_instances) == 8792
+    assert actual_instances[0].input == Input(
+        text=(
+            "Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many"
+            " clips did Natalia sell altogether in April and May?"
+        )
+    )
+    assert actual_instances[0].references == [
+        Reference(
+            output=Output(
+                text=(
+                    "Natalia sold 48/2 = <<48/2=24>>24 clips in May. Natalia sold 48+24 = <<48+24=72>>72 clips"
+                    " altogether in April and May. The answer is 72."
+                )
+            ),
+            tags=["correct"],
+        )
+    ]
+    assert actual_instances[0].split == "train"

helm/benchmark/scenarios/test_healtha_br_scenario.py

@@ -0,0 +1,57 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.healthqa_br_scenario import HEALTHQA_BR_Scenario
+from helm.benchmark.scenarios.scenario import TEST_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_healthqa_br_instance():
+    scenario = HEALTHQA_BR_Scenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+
+    instance = instances[35]
+
+    assert instance.split == TEST_SPLIT
+
+    assert instance.input.text.startswith("Homem de 22 anos de idade procura a Unidade Básica")
+
+    assert instance.references == [
+        Reference(
+            output=Output(
+                text="administração de relaxante muscular, colocando o paciente em posição de Trendelenburg, com "
+                "tentativa de redução do volume."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="encaminhamento do paciente ao Serviço de Urgência do Hospital com o pedido de avaliação "
+                "imediata do cirurgião."
+            ),
+            tags=[CORRECT_TAG],
+        ),
+        Reference(
+            output=Output(
+                text="tentativa de redução manual do aumento de volume da região inguinescrotal para a cavidade "
+                "abdominal."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="transiluminação do escroto para tentar diferenciar hérnia inguinal de hidrocele comunicante."
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(text="prescrição de antiemético e solicitação de ecografia da região inguinescrotal."),
+            tags=[],
+        ),
+    ]
+
+    correct_refs = [ref for ref in instance.references if CORRECT_TAG in ref.tags]
+    assert len(correct_refs) == 1
+
+    assert instance.references[1].is_correct

helm/benchmark/scenarios/test_ifeval_scenario.py

@@ -0,0 +1,36 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.ifeval_scenario import IFEvalScenario
+from helm.benchmark.scenarios.scenario import Input, TEST_SPLIT
+
+
+@pytest.mark.scenarios
+def test_ifeval_scenario_get_instances():
+    ifeval_scenario = IFEvalScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = ifeval_scenario.get_instances(tmpdir)
+    assert len(instances) == 541
+    assert instances[0].input == Input(
+        text=(
+            "Write a 300+ word summary of the wikipedia page "
+            '"https://en.wikipedia.org/wiki/Raymond_III,_Count_of_Tripoli". '
+            "Do not use any commas and highlight at least 3 sections that has titles in markdown format, "
+            "for example *highlighted section part 1*, *highlighted section part 2*, "
+            "*highlighted section part 3*."
+        )
+    )
+    assert instances[0].split == TEST_SPLIT
+    assert instances[0].extra_data
+    assert instances[0].extra_data["instruction_ids"] == [
+        "punctuation:no_comma",
+        "detectable_format:number_highlighted_sections",
+        "length_constraints:number_words",
+    ]
+    kwargs_groups = instances[0].extra_data["instruction_kwargs"]
+    assert all(_ is None for _ in kwargs_groups[0].values())
+    assert kwargs_groups[1]["num_highlights"] == 3
+    assert all(kwargs_groups[1][key] is None for key in kwargs_groups[1] if key != "num_highlights")
+    assert kwargs_groups[2]["relation"] == "at least"
+    assert kwargs_groups[2]["num_words"] == 300
+    assert all(kwargs_groups[2][key] is None for key in kwargs_groups[2] if key not in {"relation", "num_words"})

helm/benchmark/scenarios/test_imdb_ptbr_scenario.py

@@ -0,0 +1,27 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.imdb_ptbr_scenario import IMDB_PTBRScenario
+from helm.benchmark.scenarios.scenario import TRAIN_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_imdb_ptbr_scenario():
+    imdb_ptbr = IMDB_PTBRScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = imdb_ptbr.get_instances(tmpdir)
+    assert len(instances) == 30000
+    assert instances[0].split == TRAIN_SPLIT
+
+    assert instances[10].input.text.startswith(
+        "Foi ótimo ver algumas das minhas estrelas favoritas de 30 anos atrás, "
+        "incluindo John Ritter, Ben Gazarra e Audrey Hepburn."
+    )
+    assert len(instances[10].input.text) == 1549
+
+    assert instances[10].references == [
+        Reference(
+            output=Output(text="negativo"),
+            tags=[CORRECT_TAG],
+        )
+    ]

helm/benchmark/scenarios/test_infinite_bench_en_qa_scenario.py

@@ -0,0 +1,18 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.infinite_bench_en_qa_scenario import InfiniteBenchEnQAScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+@pytest.mark.scenarios
+def test_infinite_bench_en_qa_scenario():
+    with TemporaryDirectory() as tmpdir:
+        scenario = InfiniteBenchEnQAScenario(max_num_words=10000000)
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 351
+        assert instances[0].split == "test"
+        assert len(instances[0].input.text) == 381829
+        assert len(instances[0].references) == 1
+        assert len(instances[0].references[0].output.text) == 8
+        assert instances[0].references[0].tags == [CORRECT_TAG]

helm/benchmark/scenarios/test_infinite_bench_en_sum_scenario.py

@@ -0,0 +1,31 @@
+import pytest
+import re
+from tempfile import TemporaryDirectory
+from helm.benchmark.scenarios.infinite_bench_en_sum_scenario import InfiniteBenchEnSumScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+def count_words(text: str) -> int:
+    return len(re.split(r"\s+", text.strip()))
+
+
+@pytest.mark.scenarios
+def test_infinite_bench_en_sum_scenario():
+    with TemporaryDirectory() as tmpdir:
+        scenario = InfiniteBenchEnSumScenario(max_num_words=10000000)
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 103
+        assert instances[0].split == "test"
+        assert len(instances[0].input.text) == 1745528
+        references = instances[0].references
+        assert len(references[0].output.text) == 2865
+        assert references[0].tags == [CORRECT_TAG]
+
+        scenario = InfiniteBenchEnSumScenario(max_num_words=100000)
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 48
+        assert instances[0].split == "test"
+        assert len(instances[0].input.text) == 381778
+        references = instances[0].references
+        assert len(references[0].output.text) == 4217
+        assert references[0].tags == [CORRECT_TAG]

helm/benchmark/scenarios/test_legalbench_scenario.py

@@ -0,0 +1,30 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.legalbench_scenario import LegalBenchScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_legalbench_scenario():
+    scenario = LegalBenchScenario(subset="abercrombie")
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 100
+    assert instances[0].input == Input(text='Description: The mark "Ivory" for a product made of elephant tusks.')
+    assert instances[0].references == [
+        Reference(output=Output(text="generic"), tags=["correct"]),
+    ]
+    assert instances[0].split == "train"
+
+    scenario = LegalBenchScenario(subset="proa")
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 100
+    assert instances[0].input == Input(
+        text="Statute: Amendments to pleadings must be filed within 15 days of the filing of the initial pleading."
+    )
+    assert instances[0].references == [
+        Reference(output=Output(text="No"), tags=[CORRECT_TAG]),
+    ]
+    assert instances[0].split == "train"

helm/benchmark/scenarios/test_math_scenario.py

@@ -1,16 +1,17 @@
+import pytest
 from tempfile import TemporaryDirectory
 
 from helm.benchmark.scenarios.math_scenario import MATHScenario
 from helm.benchmark.scenarios.scenario import Input, Output, Reference
 
 
+@pytest.mark.skip(reason="competition_math has been disabled on Hugging Face Hub due to a DMCA takedown")
+@pytest.mark.scenarios
 def test_math_scenario_get_instances():
     math_scenario = MATHScenario(subject="number_theory", level="1")
     with TemporaryDirectory() as tmpdir:
         actual_instances = math_scenario.get_instances(tmpdir)
     assert len(actual_instances) == 77
     assert actual_instances[0].input == Input(text="What is the remainder when (99)(101) is divided by 9?")
-    assert actual_instances[0].references == [
-        Reference(output=Output(text="0", multimedia_content=None), tags=["correct"])
-    ]
+    assert actual_instances[0].references == [Reference(output=Output(text="0"), tags=["correct"])]
     assert actual_instances[0].split == "train"

helm/benchmark/scenarios/test_med_qa_scenario.py

@@ -0,0 +1,30 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.med_qa_scenario import MedQAScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_med_qa_scenario():
+    scenario = MedQAScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 12723
+    assert instances[0].input == Input(
+        text=(
+            "A 23-year-old pregnant woman at 22 weeks gestation presents with burning upon urination. She states it"
+            " started 1 day ago and has been worsening despite drinking more water and taking cranberry extract. She"
+            " otherwise feels well and is followed by a doctor for her pregnancy. Her temperature is 97.7°F (36.5°C),"
+            " blood pressure is 122/77 mmHg, pulse is 80/min, respirations are 19/min, and oxygen saturation is 98% on"
+            " room air. Physical exam is notable for an absence of costovertebral angle tenderness and a gravid uterus."
+            " Which of the following is the best treatment for this patient?"
+        )
+    )
+    assert instances[0].references == [
+        Reference(output=Output(text="Ampicillin"), tags=[]),
+        Reference(output=Output(text="Ceftriaxone"), tags=[]),
+        Reference(output=Output(text="Doxycycline"), tags=[]),
+        Reference(output=Output(text="Nitrofurantoin"), tags=[CORRECT_TAG]),
+    ]
+    assert instances[0].split == "train"