crfm-helm 0.5.6__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/scenarios/audio_language/ultra_suite_asr_transcription_scenario.py

@@ -0,0 +1,99 @@
+from typing import List, Tuple
+import os
+import json
+
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from huggingface_hub import snapshot_download
+
+
+def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
+    """
+    Find all pairs of MP3 and JSON files in the given directory and its subdirectories.
+    Each pair consists of an MP3 file and its corresponding JSON file with the same base name.
+
+    Args:
+        directory: Path to the directory containing the files
+
+    Returns:
+        List of tuples where each tuple contains (mp3_path, json_path)
+    """
+    pairs = []
+
+    # Walk through all directories and subdirectories
+    for root, _, files in os.walk(directory):
+        # Get all MP3 files in current directory
+        mp3_files = [f for f in files if f.endswith(".mp3")]
+
+        for mp3_file in mp3_files:
+            base_name = os.path.splitext(mp3_file)[0]
+            json_file = f"{base_name}.json"
+
+            # Check if corresponding JSON file exists in the same directory
+            if json_file in files:
+                mp3_path = os.path.join(root, mp3_file)
+                json_path = os.path.join(root, json_file)
+                pairs.append((mp3_path, json_path))
+
+    return pairs
+
+
+class UltraSuiteASRTranscriptionScenario(Scenario):
+    """
+    A scenario for evaluating the transcription capabilities of ASR systems.
+    The audio files contain speech from children, potentially with an adult present.
+    The task is to classify whether the child speaker is typically developing or has a speech disorder.
+    """
+
+    name = "speech_disorder"
+    description = "A scenario for evaluating speech disorders in children"
+    tags = ["audio", "transcription", "speech_disorder", "asr"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Create instances from the audio files and their corresponding JSON annotations.
+        The data directory should contain:
+        - Audio files (e.g., .mp3)
+        - A JSON file with annotations containing 'answer' field
+        """
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
+
+        instances: List[Instance] = []
+        split: str = TEST_SPLIT
+
+        # Find all pairs of audio and JSON files
+        pairs = find_audio_json_pairs(data_path)
+
+        for audio_path, json_path in tqdm(pairs):
+
+            # Load the annotation
+            with open(json_path, "r") as f:
+                annotation = json.load(f)
+
+            # Create references for the transcription
+            references: List[Reference] = [Reference(Output(text=annotation["transcription"]), tags=[CORRECT_TAG])]
+
+            # Create the input with audio and instruction
+            content = [
+                MediaObject(content_type="audio/mpeg", location=audio_path),
+            ]
+
+            input = Input(multimedia_content=MultimediaObject(content))
+            instances.append(Instance(input=input, references=references, split=split))
+
+        return instances

helm/benchmark/scenarios/audio_language/ultra_suite_classification_scenario.py

@@ -14,7 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
 )
 from helm.common.media_object import MediaObject, MultimediaObject
-from helm.common.general import ensure_file_downloaded
+from huggingface_hub import snapshot_download
 
 
 def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
@@ -61,7 +61,6 @@ class UltraSuiteClassificationScenario(Scenario):
     name = "speech_disorder"
     description = "A scenario for evaluating speech disorders in children"
     tags = ["audio", "classification", "speech_disorder"]
-    HF_MAPPING_URL = "https://https://huggingface.co/datasets/SAA-Lab/SLPHelmUltraSuite"
 
     def get_instruction(self, words: str) -> str:
         return f"""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. The prompt the child is trying to repeat is as follows: {words}. Based on your professional expertise: 1. Assess the child's speech in the recording for signs of typical development or potential speech-language disorder. 2. Conclude your analysis with one of the following labels only: 'typically_developing' or 'speech_disorder'. 3. Provide your response as a single letter without any additional explanation, commentary, or unnecessary text."""  # noqa: E501
@@ -74,14 +73,18 @@ class UltraSuiteClassificationScenario(Scenario):
         - A JSON file with annotations containing 'answer' field
         """
 
-        print(f"Downloading dataset from {UltraSuiteClassificationScenario.HF_MAPPING_URL} to {output_path}")
-        ensure_file_downloaded(source_url=UltraSuiteClassificationScenario.HF_MAPPING_URL, target_path=output_path)
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
 
         instances: List[Instance] = []
         split: str = TEST_SPLIT
 
         # Find all pairs of audio and JSON files
-        pairs = find_audio_json_pairs(output_path)
+        pairs = find_audio_json_pairs(data_path)
         print(f"Num pairs: {len(pairs)}")
 
         for audio_path, json_path in tqdm(pairs):
@@ -94,9 +97,14 @@ class UltraSuiteClassificationScenario(Scenario):
             words = annotation["transcription"]
             # Create references for each option
             references: List[Reference] = []
+            correct_label = 0
             for option in ["typically_developing", "speech_disorder"]:
                 reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == answer else [])
                 references.append(reference)
+                if option == answer:
+                    correct_label += 1
+            if correct_label == 0:
+                continue
 
             # Create the input with audio and instruction
             content = [

helm/benchmark/scenarios/audio_language/ultra_suite_disorder_breakdown_scenario.py

@@ -13,7 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
 )
 from helm.common.media_object import MediaObject, MultimediaObject
-from helm.common.general import ensure_file_downloaded
+from huggingface_hub import snapshot_download
 from .ultra_suite_classification_scenario import find_audio_json_pairs
 
 
@@ -27,7 +27,6 @@ class UltraSuiteDisorderBreakdownScenario(Scenario):
     name = "speech_disorder"
     description = "A scenario for evaluating and classifying specific types of speech disorders in children"
     tags = ["audio", "classification", "speech_disorder", "disorder_breakdown"]
-    HF_MAPPING_URL = "https://https://huggingface.co/datasets/SAA-Lab/SLPHelmManualLabels"
 
     def get_instruction(self, words: str) -> str:
         return f"""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. The prompt text the child is trying to repeat is as follows: {words}. Based on your professional expertise: 1. Assess the child's speech in the recording for signs of typical development or potential speech-language disorder. 2. Conclude your analysis with one of the following labels only: A - 'typically developing' (child's speech patterns and development are within normal age-appropriate ranges), B - 'articulation' (difficulty producing specific speech sounds correctly, such as substituting, omitting, or distorting sounds), C - 'phonological' (difficulty understanding and using the sound system of language, affecting sounds of a particular type). 3. Provide your response as a single letter without any additional explanation, commentary, or unnecessary text."""  # noqa: E501
@@ -39,14 +38,18 @@ class UltraSuiteDisorderBreakdownScenario(Scenario):
         - Audio files (e.g., .mp3)
         - A JSON file with annotations containing 'disorder_class' field
         """
-        print(f"Downloading dataset from {UltraSuiteDisorderBreakdownScenario.HF_MAPPING_URL} to {output_path}")
-        ensure_file_downloaded(source_url=UltraSuiteDisorderBreakdownScenario.HF_MAPPING_URL, target_path=output_path)
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
 
         instances: List[Instance] = []
         split: str = TEST_SPLIT
 
         # Find all pairs of audio and JSON files
-        pairs = find_audio_json_pairs(output_path)
+        pairs = find_audio_json_pairs(data_path)
         print(f"Num pairs: {len(pairs)}")
 
         for audio_path, json_path in tqdm(pairs):
@@ -62,9 +65,14 @@ class UltraSuiteDisorderBreakdownScenario(Scenario):
 
             # Create references for each option
             references: List[Reference] = []
+            correct_label = 0
             for option in ["typically_developing", "articulation", "phonological"]:
                 reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == label else [])
                 references.append(reference)
+                if option == label:
+                    correct_label += 1
+            if correct_label == 0:
+                continue
 
             # Create the input with audio and instruction
             content = [

helm/benchmark/scenarios/audio_language/ultra_suite_disorder_symptoms_scenario.py

@@ -14,7 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
 )
 from helm.common.media_object import MediaObject, MultimediaObject
-from helm.common.general import ensure_file_downloaded
+from huggingface_hub import snapshot_download
 
 
 def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
@@ -57,7 +57,6 @@ class UltraSuiteDisorderSymptomsScenario(Scenario):
     name = "speech_disorder"
     description = "A scenario for evaluating speech disorders in children"
     tags = ["audio", "classification", "speech_disorder"]
-    HF_MAPPING_URL = "https://https://huggingface.co/datasets/SAA-Lab/SLPHelmManualLabels"
 
     def get_instruction(self, words: str) -> str:
         prompt = f"""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. The prompt the child is trying to repeat is as follows: {words}. Based on your professional expertise: 1. Assess the child's speech in the recording and recognize any abnormal features in the child's speech. 2. These features can be on of the following: A - 'substitution', B - 'omission', C - 'addition', D - 'typically_developing', or E - 'stuttering'. Here, 'substitution' is when the child substitutes one word/phrase/syllable for another. 'omission' is when the child omits one word/phrase/syllable. 'addition' is when the child adds one word/phrase/syllable. 'typically_developing' is when the child's speech is typical of a child of their age. 'stuttering' is when the child stutters, has difficulty speaking, repeats sounds/words or prolongs sounds/words. 3. Provide your response as a single letter without any additional explanation, commentary, or unnecessary text."""  # noqa: E501
@@ -71,14 +70,18 @@ class UltraSuiteDisorderSymptomsScenario(Scenario):
         - Audio files (e.g., .mp3)
         - A JSON file with annotations containing 'answer' field
         """
-        print(f"Downloading dataset from {UltraSuiteDisorderSymptomsScenario.HF_MAPPING_URL} to {output_path}")
-        ensure_file_downloaded(source_url=UltraSuiteDisorderSymptomsScenario.HF_MAPPING_URL, target_path=output_path)
+        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
+        data_path = snapshot_download(
+            repo_id="SAA-Lab/SLPHelmManualLabels",
+            repo_type="dataset",
+            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
+        )
 
         instances: List[Instance] = []
         split: str = TEST_SPLIT
 
         # Find all pairs of audio and JSON files
-        pairs = find_audio_json_pairs(output_path)
+        pairs = find_audio_json_pairs(data_path)
 
         for audio_path, json_path in tqdm(pairs):
 
@@ -93,9 +96,14 @@ class UltraSuiteDisorderSymptomsScenario(Scenario):
             prompt = annotation["transcription"]
             # Create references for each option
             references: List[Reference] = []
+            correct_label = 0
             for option in ["substitution", "omission", "addition", "typically_developing", "stuttering"]:
                 reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == label else [])
                 references.append(reference)
+                if option == label:
+                    correct_label += 1
+            if correct_label == 0:
+                continue
 
             # Create the input with audio and instruction
             content = [

helm/benchmark/scenarios/bluex_scenario.py

@@ -0,0 +1,66 @@
+from typing import Any, List
+from pathlib import Path
+from datasets import load_dataset
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class BLUEX_Scenario(Scenario):
+    """
+    The BLUEX dataset is a benchmark used for evaluating natural language processing models in Brazilian Portuguese.
+    It consists of multiple-choice questions taken from official entrance exams of Unicamp (Convest) and USP (Fuvest),
+    covering various high school subjects. The questions include both textual prompts and visual elements. This dataset
+    was developed to assess the performance of models on tasks involving comprehension and reasoning, with a specific
+    focus on texts and exams originally written in Portuguese.
+    """
+
+    name = "bluex"
+    description = "MQA benchmark with questions from Brazilian entrance exams"
+    tags = ["knowledge", "multiple_choice", "pt-br"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the raw data and read all the dialogues
+        dataset: Any
+        # Read all the instances
+        instances: List[Instance] = []
+        cache_dir = str(Path(output_path) / "data")
+
+        dataset = load_dataset("portuguese-benchmark-datasets/BLUEX", cache_dir=cache_dir)
+        for example in dataset["questions"]:
+            # This scenario disregards issues with images
+            if example["has_associated_images"]:
+                continue
+            question = example["question"]
+            choices = example["alternatives"]
+            answer = example["answer"]
+
+            answers_dict = {}
+            for alt in choices:
+                if ")" in alt:
+                    label, text = alt.split(")", 1)
+                    label = label.strip().upper()
+                    text = text.strip()
+                    answers_dict[label] = text
+
+            if answer not in answers_dict:
+                continue
+
+            correct_answer = answers_dict[answer]
+
+            def answer_to_reference(answer: str) -> Reference:
+                return Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+
+            instance = Instance(
+                input=Input(text=question),
+                split=TEST_SPLIT,
+                references=[answer_to_reference(text) for text in answers_dict.values()],
+            )
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/cleva_scenario.py

@@ -1549,7 +1549,7 @@ class CLEVALanguageModelingScenario(CLEVAScenario):
 
 
 class CLEVACodeSynthesisScenario(CLEVAScenario):
-    """
+    r"""
     The code synthesis task of CLEVA benchmark.
 
     An example is:

helm/benchmark/scenarios/codeinsights_code_efficiency_scenario.py

@@ -0,0 +1,197 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsCodeEfficiencyScenario(Scenario):
+    name = "codeinsights_code_efficiency"
+    description = "Evaluate runtime efficiency alignment between LLM-generated code and student code"
+    tags = ["codeinsights", "c++", "code_efficiency"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario4_data.csv")
+
+        # Load test cases (unit tests)
+        test_cases = self._load_test_cases()
+
+        # Get available question IDs with test cases
+        available_question_ids = set()
+        if test_cases:
+            available_question_ids = set(test_cases.keys())
+            print(f"Loaded test cases for {len(available_question_ids)} questions")
+        else:
+            print("WARNING: No test cases loaded!")
+            return []
+
+        instances = []
+        skipped_no_tests = 0
+        skipped_insufficient_data = 0
+
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values("timestamp")
+            if len(student_df) < 4:
+                skipped_insufficient_data += 1
+                continue
+
+            first = student_df.iloc[0]
+            second = student_df.iloc[1]
+            third = student_df.iloc[2]
+            target = student_df.iloc[3]
+
+            # Check if target question has test cases BEFORE processing
+            target_question_id = target.get("question_unittest_id", None)
+            if not target_question_id or str(target_question_id) not in available_question_ids:
+                skipped_no_tests += 1
+                print(f"SKIPPING Student {student_id}, Question {target_question_id}: No test cases available")
+                continue
+
+            # Get test cases for this question (we know they exist now)
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                print(f"SKIPPING Student {student_id}, Question {target_question_id}: Empty test cases")
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            # Get student pass pattern for the target question
+            student_correctness_pattern = target.get("pass", None)
+            if student_correctness_pattern is not None:
+                main_part = int(student_correctness_pattern)
+                # Convert each character to an int
+                student_correctness_list = [int(ch) for ch in str(main_part)]
+            else:
+                student_correctness_list = []
+
+            print(f"\n=== ACCEPTED INSTANCE: Student {student_id}, Question {target_question_id} ===")
+            print(f"Test cases loaded: {len(question_test_cases)}")
+            print(f"Student correctness pattern: {student_correctness_list}")
+            print(f"Question name: {target.get('question_name', 'MISSING')}")
+
+            prompt = (
+                f"Week: {target['week']}\n"
+                f"Topic: {target['topic']}\n\n"
+                "Example 1:\n"
+                f"Question: {first['question_name']} — {first['question_text']}\n"
+                "Template:\n"
+                f"{first['question_template']}\n"
+                "Your Code:\n"
+                f"{first['response']}\n\n"
+                "Example 2:\n"
+                f"Question: {second['question_name']} — {second['question_text']}\n"
+                "Template:\n"
+                f"{second['question_template']}\n"
+                "Your Code:\n"
+                f"{second['response']}\n\n"
+                "Example 3:\n"
+                f"Question: {third['question_name']} — {third['question_text']}\n"
+                "Template:\n"
+                f"{third['question_template']}\n"
+                "Your Code:\n"
+                f"{third['response']}\n\n"
+                "Now, using that same student's coding style, attempt this:\n"
+                "Ensure that the code works perfectly, but its efficiency should be based on students' past examples.\n"
+                "If a student has a tendency to write correct but inefficient code, imitate the inefficiency "
+                "but if they write efficiently, write efficiently too.\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code is correct, includes any class definition when needed, and handles all edge cases properly. "
+                "Return the code in C++ code block format, and nothing else, and produce only one set of code."
+            )
+
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(target_question_id),
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+
+        # Print summary statistics
+        print("\n=== INSTANCE CREATION SUMMARY ===")
+        print(f"Total instances created: {len(instances)}")
+        print(f"Skipped (insufficient data): {skipped_insufficient_data}")
+        print(f"Skipped (no test cases): {skipped_no_tests}")
+        print(f"Available test case question IDs: {len(available_question_ids)}")
+
+        if instances:
+            print("Sample created instances:")
+            for i, inst in enumerate(instances[:5]):
+                if inst.extra_data is None:
+                    test_count = 0
+                else:
+                    test_count = len(inst.extra_data.get("test_cases", []))
+                print(f"  {inst.id}: {test_count} test cases")
+
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}

helm/benchmark/scenarios/codeinsights_correct_code_scenario.py

@@ -0,0 +1,78 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, VALID_SPLIT
+import pandas as pd
+
+
+class CodeInsightsCorrectCodeScenario(Scenario):
+    name = "codeinsights_correct_code"
+    description = "Generate correct response code for C++ programming questions"
+    tags = ["codeinsights", "c++", "correct_code"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario1_2_data.csv")
+
+        # Load test cases (unit tests)
+        instances = []
+        for question_id, question_df in df.groupby("question_unittest_id"):
+            target = question_df.iloc[0]
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            prompt = (
+                f"Question: {target['question_name']} — {target['question_text']}\n\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code is correct, efficient, includes any class definition when needed, and handles all edge cases properly. "
+                "Return the code in C++ code block format, and nothing else."
+            )
+            instances.append(
+                Instance(
+                    id=f"{question_id}",
+                    input=Input(text=prompt),
+                    references=[],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(question_id) if question_id else None,
+                        "question_name": target.get("question_name", ""),
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+        return instances