crfm-helm 0.5.4__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/scenarios/spider_scenario.py

@@ -0,0 +1,91 @@
+import json
+import os
+from typing import Dict, List
+
+from filelock import FileLock
+
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded, shell
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.bird_sql_scenario_helper import (  # type: ignore
+    generate_schema_prompt,
+)
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Scenario,
+    Instance,
+    Reference,
+    VALID_SPLIT,
+    Input,
+    Output,
+)
+
+
+def _ensure_file_unzipped(source_path: str, target_path: str):
+    with FileLock(f"{target_path}.lock"):
+        if os.path.exists(target_path):
+            hlog(f"Not decompressing {source_path} because {target_path} already exists")
+            return
+        tmp_path = target_path + ".tmp"
+        ensure_directory_exists(tmp_path)
+        shell(["unzip", source_path, "-d", tmp_path])
+        shell(["mv", tmp_path, target_path])
+
+
+class SpiderScenario(Scenario):
+    """Spider 1.0"""
+
+    name = "spider"
+    description = "spider"
+    tags = ["sql"]
+
+    INSTRUCTIONS_PROMPT = """-- Using valid SQLite, answer the following questions for the tables provided above.
+"""
+    COT_PROMPT = """
+Think step by step, then generate a single SQL query in valid SQLite syntax. Respond with only your reasoning and SQL query in the following tag-delimited format:
+
+<reasoning>
+INSERT_YOUR_REASONING_HERE
+</reasoning>
+<sql>
+INSERT_YOUR_SQL_QUERY_HERE
+</sql>"""  # noqa: E501
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_parent_path = os.path.join(output_path, "data")
+        ensure_file_downloaded(
+            "https://drive.google.com/uc?id=1403EGqzIDoHMdQF4c9Bkyl7dZLZ5Wt6J&export=download&confirm=t",
+            data_parent_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+        data_root_path = os.path.join(data_parent_path, "spider_data")
+        databases_root_path = os.path.join(data_root_path, "test_database")
+
+        database_schema_prompts: Dict[str, str] = {}
+        for database_name in os.listdir(databases_root_path):
+            database_path = os.path.join(databases_root_path, database_name, f"{database_name}.sqlite")
+            if not os.path.exists(database_path):
+                # Ignore stray ".DS_Store" directory
+                continue
+
+            database_schema_prompt = generate_schema_prompt(database_path, num_rows=None)
+            database_schema_prompts[database_name] = database_schema_prompt
+
+        instances: List[Instance] = []
+        dataset_path = os.path.join(data_root_path, "test.json")
+        dataset = json.load(open(dataset_path, "r"))
+        for row in dataset:
+            database_id: str = row["db_id"]
+            question: str = row["question"]
+            gold_sql: str = row["query"]
+
+            schema_prompt = database_schema_prompts[database_id]
+            combined_prompt = schema_prompt + "\n\n" + self.INSTRUCTIONS_PROMPT + question + self.COT_PROMPT
+            instance = Instance(
+                input=Input(text=combined_prompt),
+                references=[Reference(output=Output(text=gold_sql), tags=[CORRECT_TAG])],
+                extra_data={"db_id": row["db_id"]},
+                split=VALID_SPLIT,
+            )
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/starr_patient_instructions_scenario.py

@@ -0,0 +1,90 @@
+import os
+import csv
+from typing import List
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+
+class StarrPatientInstructionsScenario(Scenario):
+    """
+    Starr Patient Instructions is a dataset created from STARR-OMOP data, containing after-visit instructions
+    for outpatient surgeries/procedures. Each example corresponds to one surgery or procedure case (only including
+    outpatient or observation/overnight cases with discharge within 24 hours) and includes the following fields:
+
+      - Diagnosis: Why the patient needs the surgery/procedure.
+      - ActualProcedure: The surgery/procedure name.
+      - HistoryPhysicalNoteText: The History & Physical note written by the surgeon.
+      - OperativeNoteText: The report describing what was done during the surgery/procedure.
+      - DischargeInstructionNoteText: The specific after-surgery care instructions given to the patient.
+
+    The task is to generate personalized post-procedure patient instructions based on the provided case details.
+
+    Sample Synthetic Prompt:
+        Given the following case details, generate personalized after-surgery care instructions.
+
+        Diagnosis: [diagnosis text]
+        Procedure: [actual procedure text]
+        History & Physical: [H&P note text]
+        Operative Report: [operative note text]
+
+        Patient Instructions:
+    """
+
+    name = "starr_patient_instructions"
+    description = "A dataset containing case details used to generate customized post-procedure patient instructions."
+    tags = ["patient_communication", "healthcare", "instruction_generation", "surgery"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        csv_path = "/share/pi/nigam/suhana/medhelm/data/starr-omop-personalized-care-instr/dataset_cases_qc.csv"
+        # Ensure the directory for the CSV file exists.
+        ensure_directory_exists(os.path.dirname(csv_path))
+
+        instances: List[Instance] = []
+        # For now, we assign all instances to the test split (zero-shot setting).
+        split = TEST_SPLIT
+
+        with open(csv_path, "r", encoding="utf-8") as csvfile:
+            reader = csv.DictReader(csvfile)
+            for row in reader:
+                # Retrieve and strip the relevant fields.
+                qc_value = row.get("QC", "").strip().upper()
+                if qc_value != "TRUE":
+                    continue
+                diagnosis = row.get("Diagnosis", "").strip()
+                actual_procedure = row.get("ActualProcedure", "").strip()
+                history_physical = row.get("HistoryPhysicalNoteText", "").strip()
+                operative_note = row.get("OperativeNoteText", "").strip()
+                discharge_instruction = row.get("DischargeInstructionNoteText", "").strip()
+
+                # Skip the instance if any required field is missing.
+                if not (
+                    diagnosis and actual_procedure and history_physical and operative_note and discharge_instruction
+                ):
+                    continue
+
+                # Construct the input prompt by concatenating the fields.
+                input_text = (
+                    f"Diagnosis: {diagnosis}\n"
+                    f"Procedure: {actual_procedure}\n"
+                    f"History & Physical: {history_physical}\n"
+                    f"Operative Report: {operative_note}\n\n"
+                )
+
+                instances.append(
+                    Instance(
+                        input=Input(text=input_text),
+                        references=[Reference(Output(text=discharge_instruction), tags=[CORRECT_TAG])],
+                        split=split,
+                    )
+                )
+
+        return instances

helm/benchmark/scenarios/summarization_scenario.py

@@ -3,7 +3,17 @@ import pickle
 
 from typing import List, Optional
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 class SummarizationScenario(Scenario):

helm/benchmark/scenarios/sumosum_scenario.py

@@ -0,0 +1,157 @@
+import os
+import re
+from typing import Dict, List, Optional
+
+import pandas as pd
+
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+try:
+    # Needed for pandas.read_excel
+    import openpyxl  # noqa
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["ibm-enterprise-scenarios"])
+
+
+class SUMOSumScenario(Scenario):
+    """SUMO Web Claims Summarization
+
+    SUMO Web Claims Summarization is a summarization task over the climate subset from the SUMO dataset.
+    The task is to write a title based on the article contents.
+
+    Citation:
+    @inproceedings{mishra-etal-2020-generating,
+        title = "Generating Fact Checking Summaries for Web Claims",
+        author = "Mishra, Rahul  and
+        Gupta, Dhruv  and
+        Leippold, Markus",
+        editor = "Xu, Wei  and
+        Ritter, Alan  and
+        Baldwin, Tim  and
+        Rahimi, Afshin",
+        booktitle = "Proceedings of the Sixth Workshop on Noisy User-generated Text (W-NUT 2020)",
+        month = nov,
+        year = "2020",
+        address = "Online",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2020.wnut-1.12",
+        doi = "10.18653/v1/2020.wnut-1.12",
+        pages = "81--90",
+        abstract = "We present SUMO, a neural attention-based approach that learns to establish correctness of textual claims based on evidence in the form of text documents (e.g., news articles or web documents). SUMO further generates an extractive summary by presenting a diversified set of sentences from the documents that explain its decision on the correctness of the textual claim. Prior approaches to address the problem of fact checking and evidence extraction have relied on simple concatenation of claim and document word embeddings as an input to claim driven attention weight computation. This is done so as to extract salient words and sentences from the documents that help establish the correctness of the claim. However this design of claim-driven attention fails to capture the contextual information in documents properly. We improve on the prior art by using improved claim and title guided hierarchical attention to model effective contextual cues. We show the efficacy of our approach on political, healthcare, and environmental datasets.",
+    }
+    """  # noqa: E501
+
+    name = "sumosum"
+    description = "Text summarization with climate corpus"
+    tags = ["summarization", "climate"]
+
+    TRAIN_RATIO = 0.2
+    TITLE_KEY = "Title"
+    DOCUMENT_KEY = "Doc_text"
+
+    def __init__(
+        self,
+        train_filter_min_length: Optional[int] = None,
+        train_filter_max_length: Optional[int] = None,
+        test_filter_min_length: Optional[int] = None,
+        test_filter_max_length: Optional[int] = None,
+        truncate_length: Optional[int] = None,
+    ):
+        """
+        Initializes the scenario.
+        Args:
+            train_filter_min_length: Int indicating minimum length for training
+                                     documents. Train examples smaller than
+                                     train_filter_min_length tokens will be filtered out.
+            train_filter_max_length: Int indicating maximum length for training
+                                     documents. Train examples larger than
+                                     train_filter_max_length tokens will be filtered out.
+            test_filter_min_length: Int indicating minimum length for training
+                                    documents. Test examples smaller than
+                                    test_filter_min_length tokens will be filtered out.
+            test_filter_max_length: Int indicating maximum length for training
+                                    documents. Test examples larger than
+                                    test_filter_max_length tokens will be filtered out.
+            truncate_length: Int indicating the maximum length in tokens to
+                            truncate documents. Documents in all splits will be
+                            truncated to truncate_length tokens.
+                            NOTE: Whitespace tokenization is used to compute tokens.
+        """
+        super().__init__()
+        self.train_filter_min_length = train_filter_min_length
+        self.train_filter_max_length = train_filter_max_length
+        self.test_filter_min_length = test_filter_min_length
+        self.test_filter_max_length = test_filter_max_length
+        self.truncate_length = truncate_length
+
+    @staticmethod
+    def _clean_and_truncate(text: str, max_length: Optional[int] = None) -> str:
+        text = re.sub(r"\s+", " ", text)
+        return " ".join(text.split()[:max_length])
+
+    def _load_dataset(self, output_path: str) -> Dict[str, pd.DataFrame]:
+        data_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(data_dir)
+
+        source_url = "https://github.com/rahulOmishra/SUMO/raw/main/climate_claims_raw.xlsx"
+        source_file = os.path.basename(source_url)
+        target_path = os.path.join(data_dir, source_file)
+        ensure_file_downloaded(
+            source_url=source_url,
+            target_path=target_path,
+        )
+
+        # Column headers: Claim_id(int),Claim,Title,Doc_text,Label(bool)
+        target_df = pd.read_excel(target_path, skiprows=1)
+        target_df = target_df.dropna(subset=[SUMOSumScenario.TITLE_KEY, SUMOSumScenario.DOCUMENT_KEY])
+        # Remove carriage return _x000D_ in Excel string
+        target_df = target_df.replace({r"_x000D_": ""}, regex=True)
+        # target_df = target_df.replace({r"_x([0-9a-fA-F]{4})_": ""}, regex=True)
+        # Split randomly (works better than split by order)
+        train_df = target_df.sample(frac=SUMOSumScenario.TRAIN_RATIO, random_state=0)
+        test_df = target_df.drop(train_df.index).sample(frac=1, random_state=0)
+        return {TRAIN_SPLIT: train_df, TEST_SPLIT: test_df}
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset_dict = self._load_dataset(output_path)
+
+        instances: List[Instance] = []
+
+        for split, split_data in dataset_dict.items():
+            for example in split_data.itertuples():
+                document = getattr(example, SUMOSumScenario.DOCUMENT_KEY)
+                title = getattr(example, SUMOSumScenario.TITLE_KEY)
+                art_len = len(document.split())
+                if split == TEST_SPLIT:
+                    if self.test_filter_max_length and art_len > self.test_filter_max_length:
+                        continue
+                    if self.test_filter_min_length and art_len < self.test_filter_min_length:
+                        continue
+                if split == TRAIN_SPLIT:
+                    if self.train_filter_max_length and art_len > self.train_filter_max_length:
+                        continue
+                    if self.train_filter_min_length and art_len < self.train_filter_min_length:
+                        continue
+
+                document = SUMOSumScenario._clean_and_truncate(document, self.truncate_length)
+                title = SUMOSumScenario._clean_and_truncate(title)
+
+                instance = Instance(
+                    input=Input(text=document),
+                    references=[Reference(output=Output(text=title), tags=[CORRECT_TAG])],
+                    split=split,
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/synthetic_efficiency_scenario.py

@@ -2,7 +2,7 @@ import os
 from typing import List
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Reference, TEST_SPLIT, CORRECT_TAG, Input, Output
 
 NUM_INPUT_TOKENS: List[int] = [
     1,

helm/benchmark/scenarios/synthetic_reasoning_natural_scenario.py

@@ -67,7 +67,17 @@ from copy import copy
 from typing import List, Dict, Literal, Tuple
 from dataclasses import dataclass
 
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 @dataclass(frozen=True)

helm/benchmark/scenarios/synthetic_reasoning_scenario.py

@@ -25,7 +25,17 @@ The model hence is asked to do the following three tasks:
 import numpy as np
 from typing import List, Dict, Tuple
 
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 ANIMALS = ["zebra", "cobra", "stork", "penguin", "shark", "lion", "buffalo", "whale", "seal", "eagle", "horse", "rat"]
 FRUITS = ["apple", "peach", "watermelon", "banana", "grape", "kiwi", "pear", "strawberry", "blueberry", "blackberry"]

helm/benchmark/scenarios/test_bigcodebench_scenario.py

@@ -0,0 +1,26 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.bigcodebench_scenario import BigCodeBenchScenario
+from helm.benchmark.scenarios.scenario import Input, TEST_SPLIT
+
+
+@pytest.mark.scenarios
+def test_bigcodebench_scenario_get_instances():
+    bigcodebench_scenario = BigCodeBenchScenario("v0.1.2")
+    with TemporaryDirectory() as tmpdir:
+        instances = bigcodebench_scenario.get_instances(tmpdir)
+    assert len(instances) == 1140
+    assert instances[0].id == "BigCodeBench/0"
+    assert instances[0].input == Input(
+        text=(
+            "Calculates the average of the sums of absolute differences between each pair "
+            "of consecutive numbers for all permutations of a given list. Each permutation "
+            "is shuffled before calculating the differences. Args: - numbers (list): A list "
+            "of numbers. Default is numbers from 1 to 10.\nThe function should output with:\n"
+            "    float: The average of the sums of absolute differences for each shuffled permutation "
+            "of the list.\nYou should write self-contained code starting with:\n```\nimport itertools\n"
+            "from random import shuffle\ndef task_func(numbers=list(range(1, 3))):\n```"
+        )
+    )
+    assert instances[0].split == TEST_SPLIT

helm/benchmark/scenarios/test_czech_bank_qa_scenario.py

@@ -0,0 +1,18 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.czech_bank_qa_scenario import CzechBankQAScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input
+
+
+@pytest.mark.scenarios
+def test_czech_bank_qa_scenario_get_instances():
+    scenario = CzechBankQAScenario(config_name="default")
+    with TemporaryDirectory() as tmpdir:
+        actual_instances = scenario.get_instances(tmpdir)
+    assert len(actual_instances) == 30
+    assert actual_instances[0].input == Input(text="Get the total number of accounts in the system")
+    assert len(actual_instances[0].references) == 1
+    assert actual_instances[0].references[0].tags == [CORRECT_TAG]
+    assert actual_instances[0].references[0].output.text == "SELECT COUNT(*) FROM ACCOUNT"
+    assert actual_instances[0].split == "test"

helm/benchmark/scenarios/test_enem_challenge_scenario.py

@@ -0,0 +1,53 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.enem_challenge_scenario import ENEMChallengeScenario
+from helm.benchmark.scenarios.scenario import TEST_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_enem_challenge_scenario():
+    enem_scenario = ENEMChallengeScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = enem_scenario.get_instances(tmpdir)
+    assert len(instances) == 1431
+    assert instances[0].split == TEST_SPLIT
+
+    assert instances[0].input.text.startswith(
+        "A atmosfera terrestre é composta pelos gases nitrogênio (N2) e oxigênio (O2)"
+    )
+    assert len(instances[0].input.text) == 1163
+
+    assert instances[0].references == [
+        Reference(
+            output=Output(
+                text="reduzir o calor irradiado pela Terra mediante a substituição da produção primária pela industrialização refrigerada. "  # noqa: E501
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="promover a queima da biomassa vegetal, responsável pelo aumento do efeito estufa devido à produção de CH4. "  # noqa: E501
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="reduzir o desmatamento, mantendo-se, assim, o potencial da vegetação em absorver o CO2 da atmosfera. "  # noqa: E501
+            ),
+            tags=[CORRECT_TAG],
+        ),
+        Reference(
+            output=Output(
+                text="aumentar a concentração atmosférica de H2O, molécula capaz de absorver grande quantidade de calor. "  # noqa: E501
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="remover moléculas orgânicas polares da atmosfera, diminuindo a capacidade delas de reter calor. "  # noqa: E501
+            ),
+            tags=[],
+        ),
+    ]
+    assert instances[0].references[2].is_correct

helm/benchmark/scenarios/test_ewok_scenario.py

@@ -13,9 +13,13 @@ def test_ewok_scenario():
     with TemporaryDirectory() as tmpdir:
         try:
             instances = scenario.get_instances(tmpdir)
-        except DatasetNotFoundError:
+        except (DatasetNotFoundError, Exception):
+            # Catch generic Exception here because the Hugging Face Hub client can raise
+            # an authentication issue as the following exception:
+            # huggingface_hub.utils._errors.GatedRepoError: 401 Client Error.
+            # but GatedRepoError is a type in a private package, so we avoid referencing it
             pytest.skip("Unable to access gated dataset on Hugging Face Hub; skipping test")
-    assert len(instances) == 8748
+    assert len(instances) == 8750
     assert "believes" in instances[0].input.text
     assert len(instances[0].references) == 2
     assert "inside" in instances[0].references[0].output.text

helm/benchmark/scenarios/test_gold_commodity_news_scenario.py

@@ -0,0 +1,18 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.gold_commodity_news_scenario import GoldCommodityNewsScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_legalbench_scenario():
+    scenario = GoldCommodityNewsScenario(category="price_or_not")
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 11412
+    assert instances[0].input == Input(text="april gold down 20 cents to settle at $1,116.10/oz")
+    assert instances[0].references == [
+        Reference(output=Output(text="Yes"), tags=[CORRECT_TAG]),
+    ]
+    assert instances[0].split == "test"

helm/benchmark/scenarios/test_gpqa_scenario.py

@@ -0,0 +1,44 @@
+import pytest
+from tempfile import TemporaryDirectory
+from helm.benchmark.scenarios.gpqa_scenario import GPQAScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+@pytest.mark.scenarios
+def test_gpqa_scenario():
+    with TemporaryDirectory() as tmpdir:
+        scenario = GPQAScenario(subset="gpqa_main")
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 448
+        assert instances[0].split == "test"
+        assert len(instances[0].input.text) == 689
+        references = instances[0].references
+        assert len(references[0].output.text) == 10
+        assert len(references[1].output.text) == 6
+        assert len(references[2].output.text) == 9
+        assert len(references[3].output.text) == 7
+        assert references[3].tags == [CORRECT_TAG]
+
+        scenario = GPQAScenario(subset="gpqa_diamond")
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 198
+        assert instances[0].split == "test"
+        assert len(instances[0].input.text) == 262
+        references = instances[0].references
+        assert len(references[0].output.text) == 8
+        assert len(references[1].output.text) == 9
+        assert len(references[2].output.text) == 8
+        assert len(references[3].output.text) == 8
+        assert references[3].tags == [CORRECT_TAG]
+
+        scenario = GPQAScenario(subset="gpqa_extended")
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 546
+        assert instances[0].split == "test"
+        assert len(instances[0].input.text) == 689
+        references = instances[0].references
+        assert len(references[0].output.text) == 10
+        assert len(references[1].output.text) == 6
+        assert len(references[2].output.text) == 9
+        assert len(references[3].output.text) == 7
+        assert references[3].tags == [CORRECT_TAG]

helm/benchmark/scenarios/test_ifeval_scenario.py

@@ -0,0 +1,36 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.ifeval_scenario import IFEvalScenario
+from helm.benchmark.scenarios.scenario import Input, TEST_SPLIT
+
+
+@pytest.mark.scenarios
+def test_ifeval_scenario_get_instances():
+    ifeval_scenario = IFEvalScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = ifeval_scenario.get_instances(tmpdir)
+    assert len(instances) == 541
+    assert instances[0].input == Input(
+        text=(
+            "Write a 300+ word summary of the wikipedia page "
+            '"https://en.wikipedia.org/wiki/Raymond_III,_Count_of_Tripoli". '
+            "Do not use any commas and highlight at least 3 sections that has titles in markdown format, "
+            "for example *highlighted section part 1*, *highlighted section part 2*, "
+            "*highlighted section part 3*."
+        )
+    )
+    assert instances[0].split == TEST_SPLIT
+    assert instances[0].extra_data
+    assert instances[0].extra_data["instruction_ids"] == [
+        "punctuation:no_comma",
+        "detectable_format:number_highlighted_sections",
+        "length_constraints:number_words",
+    ]
+    kwargs_groups = instances[0].extra_data["instruction_kwargs"]
+    assert all(_ is None for _ in kwargs_groups[0].values())
+    assert kwargs_groups[1]["num_highlights"] == 3
+    assert all(kwargs_groups[1][key] is None for key in kwargs_groups[1] if key != "num_highlights")
+    assert kwargs_groups[2]["relation"] == "at least"
+    assert kwargs_groups[2]["num_words"] == 300
+    assert all(kwargs_groups[2][key] is None for key in kwargs_groups[2] if key not in {"relation", "num_words"})

helm/benchmark/scenarios/test_imdb_ptbr_scenario.py

@@ -0,0 +1,27 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.imdb_ptbr_scenario import IMDB_PTBRScenario
+from helm.benchmark.scenarios.scenario import TRAIN_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_imdb_ptbr_scenario():
+    imdb_ptbr = IMDB_PTBRScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = imdb_ptbr.get_instances(tmpdir)
+    assert len(instances) == 30000
+    assert instances[0].split == TRAIN_SPLIT
+
+    assert instances[10].input.text.startswith(
+        "Foi ótimo ver algumas das minhas estrelas favoritas de 30 anos atrás, "
+        "incluindo John Ritter, Ben Gazarra e Audrey Hepburn."
+    )
+    assert len(instances[10].input.text) == 1549
+
+    assert instances[10].references == [
+        Reference(
+            output=Output(text="negativo"),
+            tags=[CORRECT_TAG],
+        )
+    ]