crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/live_qa_scenario.py

@@ -0,0 +1,94 @@
+import os
+from typing import List
+from xml.etree.ElementTree import Element
+import xml.etree.ElementTree as ET
+
+from helm.common.general import ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, TEST_SPLIT, Input, Instance, Output, Reference, Scenario
+
+
+class LiveQAScenario(Scenario):
+    """
+    TREC-2017 LiveQA: Medical Question Answering Task
+
+    The LiveQA'17 medical task focuses on consumer health question answering.
+    Please refer to the original paper for more information about the constructed datasets and the LiveQA Track:
+    https://trec.nist.gov/pubs/trec26/papers/Overview-QA.pdf
+
+    Paper citation:
+
+        @inproceedings{LiveMedQA2017,
+          author    = {Asma {Ben Abacha} and Eugene Agichtein and Yuval Pinter and Dina Demner{-}Fushman},
+          title     = {Overview of the Medical Question Answering Task at TREC 2017 LiveQA},
+          booktitle = {TREC 2017},
+          year      = {2017}
+        }
+    """
+
+    SOURCE_REPO_URL = "https://raw.githubusercontent.com/abachaa/LiveQA_MedicalTask_TREC2017/master/TestDataset/"
+    FILENAME = "TREC-2017-LiveQA-Medical-Test-Questions-w-summaries.xml"
+
+    name = "live_qa"
+    description = "TREC-2017 LiveQA: Medical Question Answering Task"
+    tags = ["knowledge", "generation", "question_answering", "biomedical"]
+
+    def download_liveqa(self, path: str):
+        """Download the XML file containing the questions & reference answers"""
+        ensure_file_downloaded(
+            source_url=os.path.join(self.SOURCE_REPO_URL, self.FILENAME),
+            target_path=os.path.join(path, self.FILENAME),
+            unpack=False,
+        )
+
+    @staticmethod
+    def remove_whitespace(s: str) -> str:
+        """Just remove all whitespace from a string"""
+        return " ".join(s.strip().split())
+
+    @staticmethod
+    def _extract_question_id(element: Element):
+        return element.attrib["qid"]
+
+    @classmethod
+    def _extract_question(cls, element: Element) -> str:
+        """Given an XML Element representing a question, extract just the question as text"""
+        return cls.remove_whitespace(element.find("NLM-Summary").text)  # type: ignore
+
+    @classmethod
+    def _extract_answers(cls, element: Element) -> List[str]:
+        """Given an XML Element representing a question, extract the reference answers"""
+        answers = []
+        for answer in element.iter("ANSWER"):
+            answers.append(cls.remove_whitespace(answer.text))  # type: ignore
+
+        return answers
+
+    def process_xml(self, base_path: str) -> List[Instance]:
+        """Parse the XMLs into question-answer(s) pairs"""
+        xml_path = os.path.join(base_path, self.FILENAME)
+        tree = ET.parse(xml_path)
+        root = tree.getroot()
+
+        instances = []
+        for question_root in root:
+            # get the actual question and question ID
+            id = self._extract_question_id(question_root)
+            question = Input(self._extract_question(question_root))
+
+            # parse out the reference answers
+            answers = self._extract_answers(question_root)
+            references = [Reference(Output(answer), tags=[CORRECT_TAG]) for answer in answers]
+
+            # stitch it all together
+            instances.append(Instance(question, references, split=TEST_SPLIT, id=id))
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """entrypoint to creating this scenario's instances"""
+        # get the dataset
+        self.download_liveqa(output_path)
+
+        # get the instances by parsing the XML
+        instances = self.process_xml(output_path)
+        return instances

helm/benchmark/scenarios/lm_entry_scenario.py

@@ -0,0 +1,185 @@
+import json
+import os
+from typing import List
+
+from helm.common.general import ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Reference, Scenario, Instance, Input, TEST_SPLIT, Output
+
+
+class LMEntryScenario(Scenario):
+    """
+    The LMentry Benchmark
+    https://arxiv.org/pdf/2211.02069.pdf
+
+    The implementation is with reference to the original repo: https://github.com/aviaefrat/lmentry
+    The data is also downloaded from the repo.
+
+    LMentry evaluates LM's abilities of performing elementary language tasks. Examples include
+    finding which word is shorter, or which word is the last in a sentence.
+    """
+
+    name = "lm_entry"
+    description = "The LMentry benchmark for elementary language tasks"
+    tags: List[str] = []
+    url_template = "https://raw.githubusercontent.com/aviaefrat/lmentry/main/data/{subtask}.json"
+    task_to_subtasks = {
+        "all_words_from_category": [
+            "all_words_from_category",
+            "all_words_from_category_0_distractors",
+            "all_words_from_category_1_distractors",
+            "all_words_from_category_2_distractors",
+        ],
+        "any_words_from_category": [
+            "any_words_from_category",
+            "any_words_from_category_3_distractors",
+            "any_words_from_category_4_distractors",
+            "any_words_from_category_5_distractors",
+        ],
+        "bigger_number": ["bigger_number"],
+        # "ends_with_letter": ["ends_with_letter"], # HELM's metrics currently don't support this
+        # "ends_with_word": ["ends_with_word"], # HELM's metrics currently don't support this
+        "first_alphabetically": [
+            "first_alphabetically",
+            "first_alphabetically_consecutive_first_letter",
+            "first_alphabetically_different_first_letter",
+            "first_alphabetically_far_first_letter",
+            "first_alphabetically_same_first_letter",
+        ],
+        "first_letter": ["first_letter"],
+        "first_word": ["first_word"],
+        "homophones": ["homophones"],
+        "last_letter": ["last_letter"],
+        "last_word": ["last_word"],
+        "least_associated_word": ["least_associated_word"],
+        "less_letters": ["less_letters", "less_letters_length_diff_1", "less_letters_length_diff_3plus"],
+        "more_letters": ["more_letters", "more_letters_length_diff_1", "more_letters_length_diff_3plus"],
+        "most_associated_word": ["most_associated_word"],
+        "rhyming_word": [
+            "rhyming_word",
+            "rhyming_word_orthographically_different",
+            "rhyming_word_orthographically_similar",
+        ],
+        # "sentence_containing": ["sentence_containing"], # HELM's metrics currently don't support this
+        # "sentence_not_containing": ["sentence_not_containing"], # HELM's metrics currently don't support this
+        "smaller_number": ["smaller_number"],
+        # "starts_with_letter": ["starts_with_letter"], # HELM's metrics currently don't support this
+        # "starts_with_word": ["starts_with_word"], # HELM's metrics currently don't support this
+        "word_after": ["word_after"],
+        "word_before": ["word_before"],
+        # "word_containing": ["word_containing"], # HELM's metrics currently don't support this
+        # "word_not_containing": ["word_not_containing"], # HELM's metrics currently don't support this
+    }
+
+    def __init__(self, task: str):
+        super().__init__()
+        assert task in self.task_to_subtasks, f"Unsupported task: {task}"
+        self.task: str = task
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the raw data
+        data_paths: List[str] = []
+
+        for subtask in self.task_to_subtasks[self.task]:
+            data_path: str = os.path.join(output_path, f"{subtask}.json")
+            ensure_file_downloaded(
+                source_url=self.url_template.format(subtask=subtask),
+                target_path=data_path,
+                unpack=False,
+            )
+            data_paths.append(data_path)
+
+        def generate_references_for_multiple_choice_question(
+            options: List[str], correct_answer: str
+        ) -> List[Reference]:
+            references: List[Reference] = []
+
+            for option in options:
+                if option == correct_answer:
+                    references.append(Reference(Output(text=option), tags=[CORRECT_TAG]))
+                else:
+                    references.append(Reference(Output(text=option), tags=[]))
+
+            return references
+
+        def generate_references_for_generation_question(correct_answer: str) -> List[Reference]:
+            return generate_references_for_multiple_choice_question(
+                options=[correct_answer], correct_answer=correct_answer
+            )
+
+        instances: List[Instance] = []
+
+        for data_path in data_paths:
+            data: dict = json.load(open(data_path))
+
+            for example in data["examples"].values():
+                input_text: str = example["input"]
+
+                # Normalize the input text to the same format
+                if input_text.startswith("Q: "):
+                    input_text = input_text[3:]
+
+                if input_text.endswith("\n"):
+                    input_text = input_text[:-1]
+                elif input_text.endswith("\nA:"):
+                    input_text = input_text[:-3]
+
+                input = Input(text=input_text)
+                references: List[Reference]
+
+                if self.task == "all_words_from_category":
+                    correct_answer = "yes" if example["metadata"]["num_distractors"] == 0 else "no"
+                    references = generate_references_for_multiple_choice_question(
+                        options=["yes", "no"], correct_answer=correct_answer
+                    )
+                elif self.task == "any_words_from_category":
+                    correct_answer = "yes" if len(example["metadata"]["category_words"]) > 0 else "no"
+                    references = generate_references_for_multiple_choice_question(
+                        options=["yes", "no"], correct_answer=correct_answer
+                    )
+                elif self.task == "bigger_number" or self.task == "smaller_number":
+                    references = generate_references_for_multiple_choice_question(
+                        options=[str(example["metadata"]["n1"]), str(example["metadata"]["n2"])],
+                        correct_answer=str(example["metadata"]["answer"]),
+                    )
+                elif self.task == "first_alphabetically":
+                    references = generate_references_for_multiple_choice_question(
+                        options=[example["metadata"]["word1"], example["metadata"]["word2"]],
+                        correct_answer=example["metadata"]["answer"],
+                    )
+                elif self.task == "first_letter" or self.task == "last_letter":
+                    references = generate_references_for_generation_question(example["metadata"]["answer"])
+                elif self.task == "first_word" or self.task == "last_word":
+                    references = generate_references_for_generation_question(example["metadata"]["answer"])
+                elif self.task == "homophones":
+                    references = generate_references_for_multiple_choice_question(
+                        options=[example["metadata"]["answer"], example["metadata"]["distractor"]],
+                        correct_answer=example["metadata"]["answer"],
+                    )
+                elif self.task == "least_associated_word" or self.task == "most_associated_word":
+                    references = generate_references_for_multiple_choice_question(
+                        options=[example["metadata"]["answer"]] + example["metadata"]["distractors"],
+                        correct_answer=example["metadata"]["answer"],
+                    )
+                elif self.task == "less_letters" or self.task == "more_letters":
+                    references = generate_references_for_multiple_choice_question(
+                        options=[example["metadata"]["word1"], example["metadata"]["word2"]],
+                        correct_answer=example["metadata"]["answer"],
+                    )
+                elif self.task == "rhyming_word":
+                    references = generate_references_for_multiple_choice_question(
+                        options=[example["metadata"]["answer"], example["metadata"]["distractor"]],
+                        correct_answer=example["metadata"]["answer"],
+                    )
+                elif self.task == "word_before" or self.task == "word_after":
+                    references = generate_references_for_generation_question(example["metadata"]["answer"])
+                else:
+                    raise ValueError(f"Unsupported task: {self.task}")
+
+                instance = Instance(
+                    input=input,
+                    references=references,
+                    split=TEST_SPLIT,
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/lmkt_scenarios.py

@@ -0,0 +1,288 @@
+"""Cultural alignment evaluation scenario based on Vietnam World Values Survey responses."""
+
+import os
+import json
+import random
+from typing import List
+from datasets import load_dataset
+from huggingface_hub import snapshot_download
+
+from helm.common.hierarchical_logger import hlog, hwarn
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+SUPPORTED_LANGUAGES = ["en", "vi"]
+
+
+class CulturalValueUnderstandingWVSScenario(Scenario):
+    """Cultural values understanding evaluation based on Vietnam World Values Survey responses."""
+
+    name = "cultural_value_understanding_wvs"
+    description = "Evaluates model understanding of cultural values from WVS Wave 7"
+    tags = ["cultural_value_understanding"]
+
+    def __init__(
+        self,
+        language: str,
+        personas_filename: str = "vn_personas_with_filtered_responses.json",
+        questions_filename: str = "wvs_questions_dump.en.json",
+        repo_id: str = "stair-lab/cultural-alignment-dataset",
+        num_personas: int = 10,
+        num_question_variants: int = 2,
+        include_few_shot_examples: bool = False,
+    ):
+        super().__init__()
+        self.language = language
+        self.personas_filename = personas_filename
+        self.questions_filename = questions_filename
+        self.repo_id = repo_id
+        self.num_personas = 10 if isinstance(num_personas, str) else num_personas
+        self.num_question_variants = min(num_question_variants, 4)
+        self.include_few_shot_examples = include_few_shot_examples
+
+        if self.language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {self.language}. Supported languages are: {SUPPORTED_LANGUAGES}")
+
+    def get_few_shot_examples(self) -> str:
+        """Return a few-shot learning examples formatted as a string."""
+        examples = """Here are some examples:
+
+Example 1:
+You are a 35 year old male from Vietnam. You have higher education and consider yourself middle class. You are married.
+
+How important is family in your life?
+
+1. Very important
+2. Rather important
+3. Not very important
+4. Not at all important
+
+Please select the option number that best represents your view.
+1
+
+Example 2:
+You are a 28 year old female from Vietnam. You have middle education and consider yourself working class. You are single.
+
+Would you say that most people can be trusted or that you need to be very careful in dealing with people?
+
+1. Most people can be trusted
+2. Need to be very careful
+
+Please select the option number that best represents your view.
+2
+
+Example 3:
+You are a 50 year old male from Vietnam. You have lower education and consider yourself lower class. You are married.
+
+Do you think that homosexuality is justifiable?
+
+1. Never justifiable
+2. Rarely justifiable
+3. Sometimes justifiable
+4. Always justifiable
+
+Please select the option number that best represents your view.
+1
+
+Now answer the following question:
+"""  # noqa: E501
+        return examples
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Generate test instances from Vietnam personas and WVS questions."""
+        instances: List[Instance] = []
+
+        # Download files from Hugging Face Hub
+        repo_local_path = snapshot_download(
+            repo_id=self.repo_id, repo_type="dataset", revision="fe54b6f5d75cfca5377707cd7199e39f517e3a1f"
+        )
+
+        # Load the downloaded files
+        with open(os.path.join(repo_local_path, self.personas_filename), "r", encoding="utf-8") as f:
+            personas = json.load(f)
+
+        with open(os.path.join(repo_local_path, self.questions_filename), "r", encoding="utf-8") as f:
+            questions = json.load(f)
+
+        # Get few-shot examples
+        few_shot_examples = self.get_few_shot_examples() if self.include_few_shot_examples else ""
+
+        # Sample personas
+        sampled_personas = random.sample(personas, min(self.num_personas, len(personas)))
+
+        # Create instances for each persona and question
+        for persona in sampled_personas:
+            # Get demographic info for persona description
+            persona_desc = (
+                f"You are a {persona.get('age', 'adult')} year old {persona.get('sex', 'person')} from Vietnam. "
+            )
+            persona_desc += f"You have {persona.get('education', 'some')} education and consider yourself {persona.get('social_class', 'middle class')}. "  # noqa: E501
+            persona_desc += f"You are {persona.get('marital_status', 'single')}."
+
+            # Process each question this persona answered
+            for qid, human_response in persona.get("responses", {}).items():
+                # Skip if no human response or if it's 0 (which might be a "Don't know" response)
+                if human_response is None:
+                    continue
+
+                # Convert human_response to int (if possible)
+                try:
+                    human_response_int = int(human_response)
+                except (ValueError, TypeError):
+                    # Skip if human_response can't be converted to int
+                    continue
+
+                # Get question info
+                question_data = questions.get(qid, {})
+                if not question_data:
+                    continue
+
+                # Get options directly from question data
+                q_options = question_data.get("options", [])
+                if not q_options:
+                    continue
+
+                # Skip if human_response is out of range
+                if human_response_int < 0 or human_response_int > len(q_options):
+                    continue
+
+                # Special handling for "Don't know" or zero responses
+                if human_response_int == 0:
+                    # Some questions might encode "Don't know" as 0
+                    # Skip for now, or you could add special handling
+                    continue
+
+                # Use the predefined question variations
+                question_variants = question_data.get("questions", [])
+                if not question_variants:
+                    question_variants = [f"Question {qid}: {question_data.get('description', '')}"]
+
+                # Use the specified number of variants
+                variants_to_use = min(self.num_question_variants, len(question_variants))
+                selected_variants = question_variants[:variants_to_use]
+
+                # Create instances for each selected question variant
+                for q_text in selected_variants:
+                    # Format the prompt with or without few-shot examples
+                    if self.include_few_shot_examples:
+                        prompt = f"{few_shot_examples}{persona_desc}\n\n{q_text}\n\n"
+                    else:
+                        prompt = f"{persona_desc}\n\n{q_text}\n\n"
+
+                    # Add options from question data - with numbers, not letters
+                    for i, opt in enumerate(q_options, 1):
+                        prompt += f"{i}. {opt}\n"
+
+                    prompt += "\nPlease select the option number that best represents your view. Return only the option number. Do not return anything else."  # noqa: E501
+
+                    # Create a reference with just the human response number
+                    # We don't create multiple references, just use the actual human response
+                    reference = Reference(Output(text=str(human_response_int)), tags=[CORRECT_TAG])
+
+                    # Create the instance
+                    instance = Instance(
+                        input=Input(text=prompt),
+                        references=[reference],
+                        split=TEST_SPLIT,
+                    )
+                    instances.append(instance)
+
+        if not instances:
+            hwarn("No valid instances were created. Check the input data and parameters.")
+        else:
+            hlog(f"Created {len(instances)} instances for cultural value understanding scenario.")
+
+        return instances
+
+
+class SocialNormApplicationNormADScenario(Scenario):
+    """Social norms application evaluation based on Vietnam World Values Survey responses."""
+
+    name = "social_norm_application_normad"
+    description = "Evaluates model understanding of social norms from WVS Wave 7"
+    tags = ["social_norm_application"]
+
+    def __init__(
+        self,
+        language: str,
+    ):
+        super().__init__()
+        self.language = language
+
+        if self.language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {self.language}. Supported languages are: {SUPPORTED_LANGUAGES}")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset = load_dataset(
+            "akhilayerukola/NormAd", split="train", revision="311049f5610e932792672e4e13e7df62b6290336"
+        )
+
+        instances: List[Instance] = []
+
+        for item in dataset:
+            input_text = (
+                f"Assumming that below norms and story are from {item['Country'].capitalize()}:\n"
+                f"{item['Background']}\n\n{item['Story']}\n"
+                "Return one of the following options: 'yes', 'no', or 'neutral'. Do not return anything else.\n\n"
+            )
+
+            instance = Instance(
+                input=Input(text=input_text),
+                references=[Reference(Output(text=item["Gold Label"]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances
+
+
+class SocialNormExplanationNormADScenario(Scenario):
+    """Social norms explanation evaluation based on Vietnam World Values Survey responses."""
+
+    name = "social_norm_explanation_normad"
+    description = "Evaluates model understanding of social norms from WVS Wave 7"
+    tags = ["social_norm_explanation"]
+
+    def __init__(
+        self,
+        language: str,
+    ):
+        super().__init__()
+        self.language = language
+
+        if self.language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {self.language}. Supported languages are: {SUPPORTED_LANGUAGES}")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset = load_dataset(
+            "akhilayerukola/NormAd", split="train", revision="311049f5610e932792672e4e13e7df62b6290336"
+        )
+
+        instances: List[Instance] = []
+
+        for item in dataset:
+            if not item["Explanation"]:
+                continue
+
+            input_text = (
+                f"Assumming that below norms and story are from {item['Country'].capitalize()}:\n"
+                f"{item['Background']}\n\n{item['Story']}\n"
+                f"The answer is {item['Gold Label']}. "
+                "Briefly explain the reasoning behind this answer in one or two sentences.\n\n"
+            )
+
+            instance = Instance(
+                input=Input(text=input_text),
+                references=[Reference(Output(text=item["Explanation"]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/lsat_qa_scenario.py

@@ -2,8 +2,9 @@ import os
 import json
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,
@@ -13,6 +14,7 @@ from .scenario import (
     CORRECT_TAG,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -143,3 +145,15 @@ class LSATScenario(Scenario):
                             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="lsat_qa",
+            display_name="LSAT",
+            description="The LSAT benchmark for measuring analytical reasoning on the Law School "
+            "Admission Test (LSAT; [Zhong et al., "
+            "2021](https://arxiv.org/pdf/2104.06598.pdf)).",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/madinah_qa_scenario.py

@@ -0,0 +1,73 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MadinahQAScenario(Scenario):
+    """MadinahQA Scenario"""
+
+    name = "madinah_qa"
+    description = "Arabic language competency benchmark"
+    tags = ["language", "multiple_choice"]
+
+    OPTIONS = ["A", "B", "C", "D"]
+    HF_SPLIT_TO_HELM_SPLIT = {"dev": TRAIN_SPLIT, "test": TEST_SPLIT}
+    SUBSETS = ["Arabic Language (General)", "Arabic Language (Grammar)"]
+
+    def __init__(self, subset: str):
+        super().__init__()
+        subset = subset.replace("_", " ")
+        if subset not in self.SUBSETS:
+            raise Exception(f"Unknown subset: {subset}")
+        self.subset = subset
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        instances: List[Instance] = []
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "MBZUAI/MadinahQA",
+            self.subset,
+            revision="62e7c86ac5c07245a5a952722691d77ddb41f695",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            for row_index, row in enumerate(dataset):
+                input = Input(text=row["Question"])
+                references: List[Reference] = []
+                correct_option_index = ord(row["Answer Key"]) - ord("A") + 1
+                for option_index in range(1, 6):
+                    column_name = f"Option {option_index}"
+                    if not row[column_name]:
+                        continue
+                    references.append(
+                        Reference(
+                            output=Output(text=row[column_name]),
+                            tags=[CORRECT_TAG] if option_index == correct_option_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=f"id{row_index}",
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances