crfm-helm 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/scenarios/code_scenario.py

@@ -55,6 +55,7 @@ import os
 import sys
 from typing import List, Dict, Iterable, Optional, cast
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.code_scenario_helper import run as run_reindent
@@ -69,6 +70,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -331,3 +333,29 @@ class CodeScenario(Scenario):
             raise ValueError(f"Unknown dataset: {self.dataset}")
 
         return cast(List[Instance], instances)
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.dataset == "humaneval":
+            return ScenarioMetadata(
+                name="code_humaneval",
+                display_name="HumanEval (Code)",
+                description="The HumanEval benchmark for measuring functional correctness for synthesizing "
+                "programs from docstrings [(Chen et al., "
+                "2021)](https://arxiv.org/pdf/2107.03374.pdf).",
+                taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+                main_metric="pass",
+                main_split="test",
+            )
+        elif self.dataset == "apps":
+            return ScenarioMetadata(
+                name="code_apps",
+                display_name="APPS (Code)",
+                description="The APPS benchmark for measuring competence on code challenges [(Hendrycks et "
+                "al., "
+                "2021)](https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/hash/c24cd76e1ce41366a4bbe8a49b02a028-Abstract-round2.html).",
+                taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+                main_metric="test_avg",
+                main_split="test",
+            )
+        else:
+            raise Exception(f"Unknown dataset {self.dataset}")

helm/benchmark/scenarios/codeinsights_code_efficiency_scenario.py

@@ -0,0 +1,197 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsCodeEfficiencyScenario(Scenario):
+    name = "codeinsights_code_efficiency"
+    description = "Evaluate runtime efficiency alignment between LLM-generated code and student code"
+    tags = ["codeinsights", "c++", "code_efficiency"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario4_data.csv")
+
+        # Load test cases (unit tests)
+        test_cases = self._load_test_cases()
+
+        # Get available question IDs with test cases
+        available_question_ids = set()
+        if test_cases:
+            available_question_ids = set(test_cases.keys())
+            print(f"Loaded test cases for {len(available_question_ids)} questions")
+        else:
+            print("WARNING: No test cases loaded!")
+            return []
+
+        instances = []
+        skipped_no_tests = 0
+        skipped_insufficient_data = 0
+
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values("timestamp")
+            if len(student_df) < 4:
+                skipped_insufficient_data += 1
+                continue
+
+            first = student_df.iloc[0]
+            second = student_df.iloc[1]
+            third = student_df.iloc[2]
+            target = student_df.iloc[3]
+
+            # Check if target question has test cases BEFORE processing
+            target_question_id = target.get("question_unittest_id", None)
+            if not target_question_id or str(target_question_id) not in available_question_ids:
+                skipped_no_tests += 1
+                print(f"SKIPPING Student {student_id}, Question {target_question_id}: No test cases available")
+                continue
+
+            # Get test cases for this question (we know they exist now)
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                print(f"SKIPPING Student {student_id}, Question {target_question_id}: Empty test cases")
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            # Get student pass pattern for the target question
+            student_correctness_pattern = target.get("pass", None)
+            if student_correctness_pattern is not None:
+                main_part = int(student_correctness_pattern)
+                # Convert each character to an int
+                student_correctness_list = [int(ch) for ch in str(main_part)]
+            else:
+                student_correctness_list = []
+
+            print(f"\n=== ACCEPTED INSTANCE: Student {student_id}, Question {target_question_id} ===")
+            print(f"Test cases loaded: {len(question_test_cases)}")
+            print(f"Student correctness pattern: {student_correctness_list}")
+            print(f"Question name: {target.get('question_name', 'MISSING')}")
+
+            prompt = (
+                f"Week: {target['week']}\n"
+                f"Topic: {target['topic']}\n\n"
+                "Example 1:\n"
+                f"Question: {first['question_name']} — {first['question_text']}\n"
+                "Template:\n"
+                f"{first['question_template']}\n"
+                "Your Code:\n"
+                f"{first['response']}\n\n"
+                "Example 2:\n"
+                f"Question: {second['question_name']} — {second['question_text']}\n"
+                "Template:\n"
+                f"{second['question_template']}\n"
+                "Your Code:\n"
+                f"{second['response']}\n\n"
+                "Example 3:\n"
+                f"Question: {third['question_name']} — {third['question_text']}\n"
+                "Template:\n"
+                f"{third['question_template']}\n"
+                "Your Code:\n"
+                f"{third['response']}\n\n"
+                "Now, using that same student's coding style, attempt this:\n"
+                "Ensure that the code works perfectly, but its efficiency should be based on students' past examples.\n"
+                "If a student has a tendency to write correct but inefficient code, imitate the inefficiency "
+                "but if they write efficiently, write efficiently too.\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code is correct, includes any class definition when needed, and handles all edge cases properly. "
+                "Return the code in C++ code block format, and nothing else, and produce only one set of code."
+            )
+
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(target_question_id),
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+
+        # Print summary statistics
+        print("\n=== INSTANCE CREATION SUMMARY ===")
+        print(f"Total instances created: {len(instances)}")
+        print(f"Skipped (insufficient data): {skipped_insufficient_data}")
+        print(f"Skipped (no test cases): {skipped_no_tests}")
+        print(f"Available test case question IDs: {len(available_question_ids)}")
+
+        if instances:
+            print("Sample created instances:")
+            for i, inst in enumerate(instances[:5]):
+                if inst.extra_data is None:
+                    test_count = 0
+                else:
+                    test_count = len(inst.extra_data.get("test_cases", []))
+                print(f"  {inst.id}: {test_count} test cases")
+
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}

helm/benchmark/scenarios/codeinsights_correct_code_scenario.py

@@ -0,0 +1,78 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, VALID_SPLIT
+import pandas as pd
+
+
+class CodeInsightsCorrectCodeScenario(Scenario):
+    name = "codeinsights_correct_code"
+    description = "Generate correct response code for C++ programming questions"
+    tags = ["codeinsights", "c++", "correct_code"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario1_2_data.csv")
+
+        # Load test cases (unit tests)
+        instances = []
+        for question_id, question_df in df.groupby("question_unittest_id"):
+            target = question_df.iloc[0]
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            prompt = (
+                f"Question: {target['question_name']} — {target['question_text']}\n\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code is correct, efficient, includes any class definition when needed, and handles all edge cases properly. "
+                "Return the code in C++ code block format, and nothing else."
+            )
+            instances.append(
+                Instance(
+                    id=f"{question_id}",
+                    input=Input(text=prompt),
+                    references=[],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(question_id) if question_id else None,
+                        "question_name": target.get("question_name", ""),
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+        return instances

helm/benchmark/scenarios/codeinsights_edge_case_scenario.py

@@ -0,0 +1,192 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsEdgeCaseScenario(Scenario):
+    name = "codeinsights_edge_case"
+    description = "Evaluate alignment in edge case failure between LLM-generated code and student code"
+    tags = ["codeinsights", "c++", "edge_case"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario5_data.csv")
+
+        student_topic = pd.read_csv(
+            "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/student_performace_by_topic.csv"
+        )
+
+        # Load test cases (unit tests)
+        test_cases = self._load_test_cases()
+
+        # Get available question IDs with test cases
+        available_question_ids = set()
+        if test_cases:
+            available_question_ids = set(test_cases.keys())
+            print(f"Loaded test cases for {len(available_question_ids)} questions")
+        else:
+            print("WARNING: No test cases loaded!")
+            return []
+
+        instances = []
+        skipped_no_tests = 0
+        skipped_insufficient_data = 0
+
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values("timestamp")
+            target = student_df.iloc[0]
+
+            # Check if target question has test cases BEFORE processing
+            target_question_id = target.get("question_unittest_id", None)
+            if not target_question_id or str(target_question_id) not in available_question_ids:
+                skipped_no_tests += 1
+                print(f"SKIPPING Student {student_id}, Question {target_question_id}: No test cases available")
+                continue
+
+            # Get test cases for this question (we know they exist now)
+            target_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                target_test_cases.append(testcase)
+
+            # Verify test cases are not empty
+            if not tc_parsing_success:
+                skipped_no_tests += 1
+                print(f"SKIPPING Student {student_id}, Question {target_question_id}: Empty test cases")
+                continue
+
+            if len(target_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                target_test_cases = target_test_cases[: self.num_testcases]
+
+            # Get student pass pattern for the target question
+            student_correctness_pattern = target.get("pass", None)
+            if student_correctness_pattern is not None:
+                main_part = int(student_correctness_pattern)
+                # Convert each character to an int
+                student_correctness_list = [int(ch) for ch in str(main_part)]
+            else:
+                student_correctness_list = []
+
+            # Student specific topic performance in previous attempts
+            student_level_prompt = f"Student {student_id} has the following performance across topics:\n"
+            topic_performance = student_topic[student_topic["student_id"] == student_id]
+            for _, row in topic_performance.iterrows():
+                topic = row["topic"]
+                pass_rate = round(row["pass_rate"], 2)
+                perfect = round(row["perfect"], 2)
+
+                student_level_prompt += (
+                    f"- For topic '{topic}', the unit test pass rate is {pass_rate}, "
+                    f"and the rate of passing all unit tests is {perfect}.\n"
+                )
+
+            print(f"\n=== ACCEPTED INSTANCE: Student {student_id}, Question {target_question_id} ===")
+            print(f"Test cases loaded: {len(target_test_cases)}")
+            print(f"Student correctness pattern: {student_correctness_list}")
+            print(f"Question name: {target.get('question_name', 'MISSING')}")
+
+            prompt = (
+                "You are analyzing a student’s likely mistakes on an upcoming programming problem.\n"
+                "Your task: **predict exactly ONE unit-test index (0-based) that the student is most likely to fail.**\n"  # noqa: E501
+                "Return *only* that integer. No explanation.\n\n"
+                "=== Student Profile ===\n"
+                f"{student_level_prompt}\n"
+                "For the given programming question, identify which unit test the student would fail considering "
+                "their past performance, as well as consideration of unit test difficulty.\n"
+                f"Week: {target['week']}\n"
+                f"Topic: {target['topic']}\n\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n"
+                f"Unit Tests: {target_test_cases}\n"
+                if target_test_cases
+                else ""
+                "Think silently about:\n"
+                "• Which test seems hardest for the given topic?\n"
+                "• Where has the student historically struggled?\n"
+                "• Any edge-cases in the tests’ inputs/outputs?\n\n"
+                "******** ANSWER FORMAT (STRICT) ********\n"
+                "<integer>\n"
+                "****************************************"
+            )
+
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": target_test_cases,
+                        "question_id": str(target_question_id),
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+
+        # Print summary statistics
+        print("\n=== INSTANCE CREATION SUMMARY ===")
+        print(f"Skipped (insufficient data): {skipped_insufficient_data}")
+        print(f"Skipped (no test cases): {skipped_no_tests}")
+        print(f"Available test case question IDs: {len(available_question_ids)}")
+
+        if len(instances) >= 5:
+            print("Sample created instances:")
+            for i, inst in enumerate(instances[:5]):
+                if inst.extra_data is None:
+                    test_count = 0
+                else:
+                    test_count = len(inst.extra_data.get("test_cases", []))
+                print(f"  {inst.id}: {test_count} test cases")
+
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}

helm/benchmark/scenarios/codeinsights_student_coding_scenario.py

@@ -0,0 +1,162 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsStudentCodingScenario(Scenario):
+    name = "codeinsights_student_coding"
+    description = "Mimic student C++ style on foundational questions"
+    tags = ["codeinsights", "c++", "student_coding"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario1_2_data.csv")
+        student_topic = pd.read_csv(
+            "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/student_performace_by_topic.csv"
+        )
+
+        instances = []
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values("timestamp")
+            if len(student_df) < 4:
+                continue
+            first = student_df.iloc[0]
+            second = student_df.iloc[1]
+            third = student_df.iloc[2]
+            target = student_df.iloc[3]
+
+            # Get test cases for this question
+            question_id = target.get("question_unittest_id", None)
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            # Get student pass (0 or 1) for the target question
+            student_correctness_pattern = target.get("pass", None)
+            main_part = int(student_correctness_pattern)  # "1111111111"
+            # Convert each character to an int
+            student_correctness_list = [int(ch) for ch in str(main_part)]  # [1,1,1,1,1,1,1,1,1,1]
+
+            # Student specific topic performance in previous attempts
+            student_level_prompt = f"Student {student_id} has the following performance across topics:\n"
+            topic_performance = student_topic[student_topic["student_id"] == student_id]
+            for _, row in topic_performance.iterrows():
+                topic = row["topic"]
+                pass_rate = round(row["pass_rate"], 2)
+                perfect = round(row["perfect"], 2)
+
+                student_level_prompt += (
+                    f"- For topic '{topic}', the unit test pass rate is {pass_rate}, "
+                    f"and the rate of passing all unit tests is {perfect}.\n"
+                )
+
+            prompt = (
+                "=== Student Profile ===\n"
+                f"{student_level_prompt}\n"
+                f"Week: {target['week']}\n"
+                f"Topic: {target['topic']}\n\n"
+                "Example 1:\n"
+                f"Question: {first['question_name']} — {first['question_text']}\n"
+                "Template:\n"
+                f"{first['question_template']}\n"
+                "Your Code:\n"
+                f"{first['response']}\n\n"
+                "Example 2:\n"
+                f"Question: {second['question_name']} — {second['question_text']}\n"
+                "Template:\n"
+                f"{second['question_template']}\n"
+                "Your Code:\n"
+                f"{second['response']}\n\n"
+                "Example 3:\n"
+                f"Question: {third['question_name']} — {third['question_text']}\n"
+                "Template:\n"
+                f"{third['question_template']}\n"
+                "Your Code:\n"
+                f"{third['response']}\n\n"
+                "Now, using that same student style, attempt this:\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code includes any class definition when needed. "
+                "Return the code in C++ code block format, and nothing else."
+            )
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(question_id) if question_id else None,
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}