crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/scenarios/shc_sei_scenario.py

@@ -0,0 +1,94 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCSEIMedScenario(Scenario):
+    """
+    This dataset contains clinical notes from primary care visit encounters
+    (in-person/telehealth and telephone) of children ages 6-11 years old with ADHD
+    seen at Stanford's community-based primary care network, Packard Children's Health Alliance,
+    between 2015-2022. All children in this dataset were prescribed at least once an ADHD
+    medication (stimulants or non-stimulants) by a primary care clinician. In this
+    classification task, the LLM is tasked with classifying whether the note contains
+    documentation of side effect monitoring (recording of absence or presence of
+    medication side effects), as recommended in clinical practice guidelines.
+    From publication: https://doi.org/10.1542/peds.2024-067223
+    """
+
+    name = "shc_sei_med"
+    description = (
+        "ADHD-MedEffects is a benchmark designed to evaluate whether clinical notes for"
+        "pediatric ADHD visits document medication side effect monitoring, which is a key recommendation"
+        "in clinical practice guidelines. The dataset supports binary classification"
+        "to detect presence or absence of side effect inquiries (SEI) within notes."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    "You are reviewing a clinical note from health records of children "
+                    "with attention deficit hyperactivity disorder (ADHD). Given the following "
+                    "definitions: side Effects Inquiry (SEI): Explicit documentation by the clinician "
+                    "asking about current side effects related to ADHD medications that the child is "
+                    "taking or documentation of specific ADHD medication side effects experienced "
+                    "by the patient. SEI does *not* include future side effects monitoring, "
+                    "such as documentation of potential ADHD medication side effects, including "
+                    "planning to follow patients to monitor side effects, explaining about "
+                    "potential side effects of an ADHD medication. These documentations are not "
+                    "categorized as SEI because they consist of a plan or an explanation about "
+                    "side effects without actual side effect monitoring taking place, and "
+                    "No Side Effects Inquiry (NSEI): No evidence of side effects monitoring. "
+                    f"Provide an answer to the following question: {question} with the following context: {context} "
+                    ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
+                    "details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[SHCSEIMedScenario] Required data file not found: '{self.data_path}'")
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCSEIMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCSEIMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/shc_sequoia_scenario.py

@@ -0,0 +1,77 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCSequoiaMedScenario(Scenario):
+    """
+    Benchmark derived from manually curated answers to several questions for Sequoia clinic referrals
+    """
+
+    name = "shc_sequoia_med"
+    description = (
+        "ClinicReferral is a benchmark that determines patient eligibility for referral to the"
+        "Sequoia Clinic based on information from palliative care notes. The dataset provides"
+        "curated decisions on referral appropriateness to assist in automating clinic workflows."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        counter = 1
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)  # , quoting=csv.QUOTE_MINIMAL
+            for row in reader:
+                question = row["question"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    f" {counter} Provide an answer to the following question: {question} with the following context:"
+                    f" {context} , Answer the question with a 'A' for yes or 'B' for no. Do not provide any "
+                    "additional details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+                counter += 1
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(
+            self.data_path, msg=f"[SHCSequoiaMedScenario] Required data file not found: '{self.data_path}'"
+        )
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCSequoiaMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCSequoiaMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/simple_safety_tests_scenario.py

@@ -1,7 +1,7 @@
 from typing import List
 from datasets import load_dataset
 
-from .scenario import Scenario, Instance, Input, TEST_SPLIT, Reference, Output
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, Reference, Output
 
 
 class SimpleSafetyTestsScenario(Scenario):

helm/benchmark/scenarios/spider_scenario.py

@@ -0,0 +1,91 @@
+import json
+import os
+from typing import Dict, List
+
+from filelock import FileLock
+
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded, shell
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.bird_sql_scenario_helper import (  # type: ignore
+    generate_schema_prompt,
+)
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Scenario,
+    Instance,
+    Reference,
+    VALID_SPLIT,
+    Input,
+    Output,
+)
+
+
+def _ensure_file_unzipped(source_path: str, target_path: str):
+    with FileLock(f"{target_path}.lock"):
+        if os.path.exists(target_path):
+            hlog(f"Not decompressing {source_path} because {target_path} already exists")
+            return
+        tmp_path = target_path + ".tmp"
+        ensure_directory_exists(tmp_path)
+        shell(["unzip", source_path, "-d", tmp_path])
+        shell(["mv", tmp_path, target_path])
+
+
+class SpiderScenario(Scenario):
+    """Spider 1.0"""
+
+    name = "spider"
+    description = "spider"
+    tags = ["sql"]
+
+    INSTRUCTIONS_PROMPT = """-- Using valid SQLite, answer the following questions for the tables provided above.
+"""
+    COT_PROMPT = """
+Think step by step, then generate a single SQL query in valid SQLite syntax. Respond with only your reasoning and SQL query in the following tag-delimited format:
+
+<reasoning>
+INSERT_YOUR_REASONING_HERE
+</reasoning>
+<sql>
+INSERT_YOUR_SQL_QUERY_HERE
+</sql>"""  # noqa: E501
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_parent_path = os.path.join(output_path, "data")
+        ensure_file_downloaded(
+            "https://drive.google.com/uc?id=1403EGqzIDoHMdQF4c9Bkyl7dZLZ5Wt6J&export=download&confirm=t",
+            data_parent_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+        data_root_path = os.path.join(data_parent_path, "spider_data")
+        databases_root_path = os.path.join(data_root_path, "test_database")
+
+        database_schema_prompts: Dict[str, str] = {}
+        for database_name in os.listdir(databases_root_path):
+            database_path = os.path.join(databases_root_path, database_name, f"{database_name}.sqlite")
+            if not os.path.exists(database_path):
+                # Ignore stray ".DS_Store" directory
+                continue
+
+            database_schema_prompt = generate_schema_prompt(database_path, num_rows=None)
+            database_schema_prompts[database_name] = database_schema_prompt
+
+        instances: List[Instance] = []
+        dataset_path = os.path.join(data_root_path, "test.json")
+        dataset = json.load(open(dataset_path, "r"))
+        for row in dataset:
+            database_id: str = row["db_id"]
+            question: str = row["question"]
+            gold_sql: str = row["query"]
+
+            schema_prompt = database_schema_prompts[database_id]
+            combined_prompt = schema_prompt + "\n\n" + self.INSTRUCTIONS_PROMPT + question + self.COT_PROMPT
+            instance = Instance(
+                input=Input(text=combined_prompt),
+                references=[Reference(output=Output(text=gold_sql), tags=[CORRECT_TAG])],
+                extra_data={"db_id": row["db_id"]},
+                split=VALID_SPLIT,
+            )
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/starr_patient_instructions_scenario.py

@@ -0,0 +1,97 @@
+import csv
+from typing import List
+
+from helm.common.general import check_file_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+
+class StarrPatientInstructionsScenario(Scenario):
+    """
+    Starr Patient Instructions is a dataset created from STARR-OMOP data, containing after-visit instructions
+    for outpatient surgeries/procedures. Each example corresponds to one surgery or procedure case (only including
+    outpatient or observation/overnight cases with discharge within 24 hours) and includes the following fields:
+
+      - Diagnosis: Why the patient needs the surgery/procedure.
+      - ActualProcedure: The surgery/procedure name.
+      - HistoryPhysicalNoteText: The History & Physical note written by the surgeon.
+      - OperativeNoteText: The report describing what was done during the surgery/procedure.
+      - DischargeInstructionNoteText: The specific after-surgery care instructions given to the patient.
+
+    The task is to generate personalized post-procedure patient instructions based on the provided case details.
+
+    Sample Synthetic Prompt:
+        Given the following case details, generate personalized after-surgery care instructions.
+
+        Diagnosis: [diagnosis text]
+        Procedure: [actual procedure text]
+        History & Physical: [H&P note text]
+        Operative Report: [operative note text]
+
+        Patient Instructions:
+    """
+
+    name = "starr_patient_instructions"
+    description = (
+        "PatientInstruct is a benchmark designed to evaluate models on generating personalized"
+        "post-procedure instructions for patients. It includes real-world patient History & Physical"
+        "Note (H&P) and operative report, from which models must produce clear, actionable instructions"
+        "appropriate for patients recovering from medical interventions."
+    )
+    tags = ["patient_communication", "healthcare", "instruction_generation", "surgery"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(
+            self.data_path, msg=f"[StarrPatientInstructiosScenario] Required data file not found: '{self.data_path}'"
+        )
+        instances: List[Instance] = []
+        # For now, we assign all instances to the test split (zero-shot setting).
+        split = TEST_SPLIT
+
+        with open(self.data_path, "r", encoding="utf-8") as csvfile:
+            reader = csv.DictReader(csvfile)
+            for row in reader:
+                # Retrieve and strip the relevant fields.
+                qc_value = row.get("QC", "").strip().upper()
+                if qc_value != "TRUE":
+                    continue
+                diagnosis = row.get("Diagnosis", "").strip()
+                actual_procedure = row.get("ActualProcedure", "").strip()
+                history_physical = row.get("HistoryPhysicalNoteText", "").strip()
+                operative_note = row.get("OperativeNoteText", "").strip()
+                discharge_instruction = row.get("DischargeInstructionNoteText", "").strip()
+
+                # Skip the instance if any required field is missing.
+                if not (
+                    diagnosis and actual_procedure and history_physical and operative_note and discharge_instruction
+                ):
+                    continue
+
+                # Construct the input prompt by concatenating the fields.
+                input_text = (
+                    f"Diagnosis: {diagnosis}\n"
+                    f"Procedure: {actual_procedure}\n"
+                    f"History & Physical: {history_physical}\n"
+                    f"Operative Report: {operative_note}\n\n"
+                )
+
+                instances.append(
+                    Instance(
+                        input=Input(text=input_text),
+                        references=[Reference(Output(text=discharge_instruction), tags=[CORRECT_TAG])],
+                        split=split,
+                    )
+                )
+
+        return instances

helm/benchmark/scenarios/summarization_scenario.py

@@ -3,7 +3,17 @@ import pickle
 
 from typing import List, Optional
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 class SummarizationScenario(Scenario):

helm/benchmark/scenarios/sumosum_scenario.py

@@ -0,0 +1,157 @@
+import os
+import re
+from typing import Dict, List, Optional
+
+import pandas as pd
+
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+try:
+    # Needed for pandas.read_excel
+    import openpyxl  # noqa
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["ibm-enterprise-scenarios"])
+
+
+class SUMOSumScenario(Scenario):
+    """SUMO Web Claims Summarization
+
+    SUMO Web Claims Summarization is a summarization task over the climate subset from the SUMO dataset.
+    The task is to write a title based on the article contents.
+
+    Citation:
+    @inproceedings{mishra-etal-2020-generating,
+        title = "Generating Fact Checking Summaries for Web Claims",
+        author = "Mishra, Rahul  and
+        Gupta, Dhruv  and
+        Leippold, Markus",
+        editor = "Xu, Wei  and
+        Ritter, Alan  and
+        Baldwin, Tim  and
+        Rahimi, Afshin",
+        booktitle = "Proceedings of the Sixth Workshop on Noisy User-generated Text (W-NUT 2020)",
+        month = nov,
+        year = "2020",
+        address = "Online",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2020.wnut-1.12",
+        doi = "10.18653/v1/2020.wnut-1.12",
+        pages = "81--90",
+        abstract = "We present SUMO, a neural attention-based approach that learns to establish correctness of textual claims based on evidence in the form of text documents (e.g., news articles or web documents). SUMO further generates an extractive summary by presenting a diversified set of sentences from the documents that explain its decision on the correctness of the textual claim. Prior approaches to address the problem of fact checking and evidence extraction have relied on simple concatenation of claim and document word embeddings as an input to claim driven attention weight computation. This is done so as to extract salient words and sentences from the documents that help establish the correctness of the claim. However this design of claim-driven attention fails to capture the contextual information in documents properly. We improve on the prior art by using improved claim and title guided hierarchical attention to model effective contextual cues. We show the efficacy of our approach on political, healthcare, and environmental datasets.",
+    }
+    """  # noqa: E501
+
+    name = "sumosum"
+    description = "Text summarization with climate corpus"
+    tags = ["summarization", "climate"]
+
+    TRAIN_RATIO = 0.2
+    TITLE_KEY = "Title"
+    DOCUMENT_KEY = "Doc_text"
+
+    def __init__(
+        self,
+        train_filter_min_length: Optional[int] = None,
+        train_filter_max_length: Optional[int] = None,
+        test_filter_min_length: Optional[int] = None,
+        test_filter_max_length: Optional[int] = None,
+        truncate_length: Optional[int] = None,
+    ):
+        """
+        Initializes the scenario.
+        Args:
+            train_filter_min_length: Int indicating minimum length for training
+                                     documents. Train examples smaller than
+                                     train_filter_min_length tokens will be filtered out.
+            train_filter_max_length: Int indicating maximum length for training
+                                     documents. Train examples larger than
+                                     train_filter_max_length tokens will be filtered out.
+            test_filter_min_length: Int indicating minimum length for training
+                                    documents. Test examples smaller than
+                                    test_filter_min_length tokens will be filtered out.
+            test_filter_max_length: Int indicating maximum length for training
+                                    documents. Test examples larger than
+                                    test_filter_max_length tokens will be filtered out.
+            truncate_length: Int indicating the maximum length in tokens to
+                            truncate documents. Documents in all splits will be
+                            truncated to truncate_length tokens.
+                            NOTE: Whitespace tokenization is used to compute tokens.
+        """
+        super().__init__()
+        self.train_filter_min_length = train_filter_min_length
+        self.train_filter_max_length = train_filter_max_length
+        self.test_filter_min_length = test_filter_min_length
+        self.test_filter_max_length = test_filter_max_length
+        self.truncate_length = truncate_length
+
+    @staticmethod
+    def _clean_and_truncate(text: str, max_length: Optional[int] = None) -> str:
+        text = re.sub(r"\s+", " ", text)
+        return " ".join(text.split()[:max_length])
+
+    def _load_dataset(self, output_path: str) -> Dict[str, pd.DataFrame]:
+        data_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(data_dir)
+
+        source_url = "https://github.com/rahulOmishra/SUMO/raw/main/climate_claims_raw.xlsx"
+        source_file = os.path.basename(source_url)
+        target_path = os.path.join(data_dir, source_file)
+        ensure_file_downloaded(
+            source_url=source_url,
+            target_path=target_path,
+        )
+
+        # Column headers: Claim_id(int),Claim,Title,Doc_text,Label(bool)
+        target_df = pd.read_excel(target_path, skiprows=1)
+        target_df = target_df.dropna(subset=[SUMOSumScenario.TITLE_KEY, SUMOSumScenario.DOCUMENT_KEY])
+        # Remove carriage return _x000D_ in Excel string
+        target_df = target_df.replace({r"_x000D_": ""}, regex=True)
+        # target_df = target_df.replace({r"_x([0-9a-fA-F]{4})_": ""}, regex=True)
+        # Split randomly (works better than split by order)
+        train_df = target_df.sample(frac=SUMOSumScenario.TRAIN_RATIO, random_state=0)
+        test_df = target_df.drop(train_df.index).sample(frac=1, random_state=0)
+        return {TRAIN_SPLIT: train_df, TEST_SPLIT: test_df}
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset_dict = self._load_dataset(output_path)
+
+        instances: List[Instance] = []
+
+        for split, split_data in dataset_dict.items():
+            for example in split_data.itertuples():
+                document = getattr(example, SUMOSumScenario.DOCUMENT_KEY)
+                title = getattr(example, SUMOSumScenario.TITLE_KEY)
+                art_len = len(document.split())
+                if split == TEST_SPLIT:
+                    if self.test_filter_max_length and art_len > self.test_filter_max_length:
+                        continue
+                    if self.test_filter_min_length and art_len < self.test_filter_min_length:
+                        continue
+                if split == TRAIN_SPLIT:
+                    if self.train_filter_max_length and art_len > self.train_filter_max_length:
+                        continue
+                    if self.train_filter_min_length and art_len < self.train_filter_min_length:
+                        continue
+
+                document = SUMOSumScenario._clean_and_truncate(document, self.truncate_length)
+                title = SUMOSumScenario._clean_and_truncate(title)
+
+                instance = Instance(
+                    input=Input(text=document),
+                    references=[Reference(output=Output(text=title), tags=[CORRECT_TAG])],
+                    split=split,
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/synthetic_efficiency_scenario.py

@@ -2,7 +2,7 @@ import os
 from typing import List
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Reference, TEST_SPLIT, CORRECT_TAG, Input, Output
 
 NUM_INPUT_TOKENS: List[int] = [
     1,

helm/benchmark/scenarios/synthetic_reasoning_natural_scenario.py

@@ -67,7 +67,17 @@ from copy import copy
 from typing import List, Dict, Literal, Tuple
 from dataclasses import dataclass
 
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 @dataclass(frozen=True)

helm/benchmark/scenarios/synthetic_reasoning_scenario.py

@@ -25,7 +25,17 @@ The model hence is asked to do the following three tasks:
 import numpy as np
 from typing import List, Dict, Tuple
 
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 ANIMALS = ["zebra", "cobra", "stork", "penguin", "shark", "lion", "buffalo", "whale", "seal", "eagle", "horse", "rat"]
 FRUITS = ["apple", "peach", "watermelon", "banana", "grape", "kiwi", "pear", "strawberry", "blueberry", "blackberry"]

helm/benchmark/scenarios/test_bigcodebench_scenario.py

@@ -0,0 +1,26 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.bigcodebench_scenario import BigCodeBenchScenario
+from helm.benchmark.scenarios.scenario import Input, TEST_SPLIT
+
+
+@pytest.mark.scenarios
+def test_bigcodebench_scenario_get_instances():
+    bigcodebench_scenario = BigCodeBenchScenario("v0.1.2")
+    with TemporaryDirectory() as tmpdir:
+        instances = bigcodebench_scenario.get_instances(tmpdir)
+    assert len(instances) == 1140
+    assert instances[0].id == "BigCodeBench/0"
+    assert instances[0].input == Input(
+        text=(
+            "Calculates the average of the sums of absolute differences between each pair "
+            "of consecutive numbers for all permutations of a given list. Each permutation "
+            "is shuffled before calculating the differences. Args: - numbers (list): A list "
+            "of numbers. Default is numbers from 1 to 10.\nThe function should output with:\n"
+            "    float: The average of the sums of absolute differences for each shuffled permutation "
+            "of the list.\nYou should write self-contained code starting with:\n```\nimport itertools\n"
+            "from random import shuffle\ndef task_func(numbers=list(range(1, 3))):\n```"
+        )
+    )
+    assert instances[0].split == TEST_SPLIT

helm/benchmark/scenarios/test_czech_bank_qa_scenario.py

@@ -0,0 +1,18 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.czech_bank_qa_scenario import CzechBankQAScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input
+
+
+@pytest.mark.scenarios
+def test_czech_bank_qa_scenario_get_instances():
+    scenario = CzechBankQAScenario(config_name="default")
+    with TemporaryDirectory() as tmpdir:
+        actual_instances = scenario.get_instances(tmpdir)
+    assert len(actual_instances) == 30
+    assert actual_instances[0].input == Input(text="Get the total number of accounts in the system")
+    assert len(actual_instances[0].references) == 1
+    assert actual_instances[0].references[0].tags == [CORRECT_TAG]
+    assert actual_instances[0].references[0].output.text == "SELECT COUNT(*) FROM ACCOUNT"
+    assert actual_instances[0].split == "test"