crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/scenarios/autobencher_capabilities_scenario.py

@@ -12,7 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
 )
 from helm.common.general import ensure_directory_exists
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hwarn
 
 
 class AutoBencherCapabilitiesScenario(Scenario):
@@ -61,7 +61,7 @@ class AutoBencherCapabilitiesScenario(Scenario):
             # References are category ID, followed by level 2, 3 and 4 category names.
             references = [Reference(output=Output(text=row["gold_answer"]), tags=[CORRECT_TAG])]
             if row["gold_answer"] is None:
-                hlog(f"WARNING: Row had no gold_answer: {row}")
+                hwarn(f"Row had no gold_answer: {row}")
                 continue
             instance = Instance(input=input, references=references, split=TEST_SPLIT)
             instances.append(instance)

helm/benchmark/scenarios/bluex_scenario.py

@@ -0,0 +1,66 @@
+from typing import Any, List
+from pathlib import Path
+from datasets import load_dataset
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class BLUEX_Scenario(Scenario):
+    """
+    The BLUEX dataset is a benchmark used for evaluating natural language processing models in Brazilian Portuguese.
+    It consists of multiple-choice questions taken from official entrance exams of Unicamp (Convest) and USP (Fuvest),
+    covering various high school subjects. The questions include both textual prompts and visual elements. This dataset
+    was developed to assess the performance of models on tasks involving comprehension and reasoning, with a specific
+    focus on texts and exams originally written in Portuguese.
+    """
+
+    name = "bluex"
+    description = "MQA benchmark with questions from Brazilian entrance exams"
+    tags = ["knowledge", "multiple_choice", "pt-br"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the raw data and read all the dialogues
+        dataset: Any
+        # Read all the instances
+        instances: List[Instance] = []
+        cache_dir = str(Path(output_path) / "data")
+
+        dataset = load_dataset("portuguese-benchmark-datasets/BLUEX", cache_dir=cache_dir)
+        for example in dataset["questions"]:
+            # This scenario disregards issues with images
+            if example["has_associated_images"]:
+                continue
+            question = example["question"]
+            choices = example["alternatives"]
+            answer = example["answer"]
+
+            answers_dict = {}
+            for alt in choices:
+                if ")" in alt:
+                    label, text = alt.split(")", 1)
+                    label = label.strip().upper()
+                    text = text.strip()
+                    answers_dict[label] = text
+
+            if answer not in answers_dict:
+                continue
+
+            correct_answer = answers_dict[answer]
+
+            def answer_to_reference(answer: str) -> Reference:
+                return Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+
+            instance = Instance(
+                input=Input(text=question),
+                split=TEST_SPLIT,
+                references=[answer_to_reference(text) for text in answers_dict.values()],
+            )
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/chw_care_plan_scenario.py

@@ -1,8 +1,7 @@
-import os
 import pandas as pd
 from typing import List
 
-from helm.common.general import ensure_directory_exists
+from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -61,28 +60,30 @@ Clinical Note:
 
 class CHWCarePlanScenario(Scenario):
     """
-    A scenario for MIMIC-IV discharge summaries where the task is to predict the ICD-10 code(s).
+    A scenario for a dataset containing free form text of a clinical health worker care plan, with the
+    associated goal being to restructure that text into a given format.
 
     - Input:  The clinical note (column "MO Note").
     - Output:  The clinical note (column "MO Note"). We will use this note as the reference for entailment.
     """
 
     name = "chw_care_plan"
-    description = "A dataset containing free form text of a clinical health worker care plan, with the \
-    associated goal being to restructure that text into a given format."
+    description = (
+        "NoteExtract is a benchmark that focuses on the structured extraction of information"
+        "from free-form clinical text. It provides care plan notes authored by health workers"
+        "and evaluates a model's ability to convert them into a predefined structured format,"
+        "such as fields for Chief Complaint and History of Present Illness. The benchmark"
+        "emphasizes faithful extraction without hallucination or inference."
+    )
     tags = ["question_answering", "biomedical"]
 
-    def __init__(self):
-        """
-        :param data_file: Path to the mimiciv_icd10.feather file.
-        """
+    def __init__(self, data_path: str):
         super().__init__()
-        self.data_file = "/share/pi/nigam/datasets/CHW_Dataset.csv"
+        self.data_path = data_path
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        ensure_directory_exists(os.path.dirname(self.data_file))
-
-        df = pd.read_csv(self.data_file)  # columns: ["text", "target", ...]
+        check_file_exists(self.data_path, msg=f"[CHWCarePlanScenario] Required data file not found: '{self.data_path}'")
+        df = pd.read_csv(self.data_path)  # columns: ["text", "target", ...]
 
         instances: List[Instance] = []
 

helm/benchmark/scenarios/clear_scenario.py

@@ -2,7 +2,7 @@ import os
 import pandas as pd
 from typing import List
 
-from helm.common.general import ensure_directory_exists
+from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -76,16 +76,21 @@ class CLEARScenario(Scenario):
         "unemployment": "unemployment",
     }
 
-    def __init__(self, condition: str):
+    def __init__(self, condition: str, data_path: str):
         """Initialize the scenario with a specific medical condition"""
         super().__init__()
 
         if condition not in self.CONDITIONS:
             raise ValueError(f"Condition '{condition}' not supported. Available conditions: {self.CONDITIONS}")
-
+        self.data_path = data_path
         self.condition = condition
         self.name = f"clear_{condition}"
-        self.description = f"A dataset for evaluating {self.CONDITION_PROMPTS[condition]} detection from patient notes with yes/no/maybe classifications."  # noqa: E501
+        self.description = (
+            "CLEAR is a benchmark designed to evaluate models on their ability to detect medical"
+            "conditions from patient notes using categorical responses. Each instance consists of"
+            "a clinical note and a target condition, requiring the model to classify the patient's"
+            "history as either affirmative, negative, or uncertain."
+        )  # noqa: E501
         self.tags = ["classification", "biomedical", condition.replace("_", "-")]
 
     def get_answer_choices(self) -> List[str]:
@@ -95,9 +100,8 @@ class CLEARScenario(Scenario):
 
     def get_instances(self, output_path: str) -> List[Instance]:
         """Load and process the data for the specified conditon."""
-        data_dir = "/share/pi/nigam/suhana/medhelm/data/CLEAR/human_labeled/"
-        excel_path = os.path.join(data_dir, f"{self.condition}.xlsx")
-        ensure_directory_exists(os.path.dirname(excel_path))
+        excel_path = os.path.join(self.data_path, f"{self.condition}.xlsx")
+        check_file_exists(excel_path, msg=f"[CLEARScenario] Required data file not found: '{excel_path}'")
 
         df = pd.read_excel(excel_path)
 

helm/benchmark/scenarios/cleva_scenario.py

@@ -1549,7 +1549,7 @@ class CLEVALanguageModelingScenario(CLEVAScenario):
 
 
 class CLEVACodeSynthesisScenario(CLEVAScenario):
-    """
+    r"""
     The code synthesis task of CLEVA benchmark.
 
     An example is:

helm/benchmark/scenarios/codeinsights_code_efficiency_scenario.py

@@ -0,0 +1,197 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsCodeEfficiencyScenario(Scenario):
+    name = "codeinsights_code_efficiency"
+    description = "Evaluate runtime efficiency alignment between LLM-generated code and student code"
+    tags = ["codeinsights", "c++", "code_efficiency"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario4_data.csv")
+
+        # Load test cases (unit tests)
+        test_cases = self._load_test_cases()
+
+        # Get available question IDs with test cases
+        available_question_ids = set()
+        if test_cases:
+            available_question_ids = set(test_cases.keys())
+            print(f"Loaded test cases for {len(available_question_ids)} questions")
+        else:
+            print("WARNING: No test cases loaded!")
+            return []
+
+        instances = []
+        skipped_no_tests = 0
+        skipped_insufficient_data = 0
+
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values("timestamp")
+            if len(student_df) < 4:
+                skipped_insufficient_data += 1
+                continue
+
+            first = student_df.iloc[0]
+            second = student_df.iloc[1]
+            third = student_df.iloc[2]
+            target = student_df.iloc[3]
+
+            # Check if target question has test cases BEFORE processing
+            target_question_id = target.get("question_unittest_id", None)
+            if not target_question_id or str(target_question_id) not in available_question_ids:
+                skipped_no_tests += 1
+                print(f"SKIPPING Student {student_id}, Question {target_question_id}: No test cases available")
+                continue
+
+            # Get test cases for this question (we know they exist now)
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                print(f"SKIPPING Student {student_id}, Question {target_question_id}: Empty test cases")
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            # Get student pass pattern for the target question
+            student_correctness_pattern = target.get("pass", None)
+            if student_correctness_pattern is not None:
+                main_part = int(student_correctness_pattern)
+                # Convert each character to an int
+                student_correctness_list = [int(ch) for ch in str(main_part)]
+            else:
+                student_correctness_list = []
+
+            print(f"\n=== ACCEPTED INSTANCE: Student {student_id}, Question {target_question_id} ===")
+            print(f"Test cases loaded: {len(question_test_cases)}")
+            print(f"Student correctness pattern: {student_correctness_list}")
+            print(f"Question name: {target.get('question_name', 'MISSING')}")
+
+            prompt = (
+                f"Week: {target['week']}\n"
+                f"Topic: {target['topic']}\n\n"
+                "Example 1:\n"
+                f"Question: {first['question_name']} — {first['question_text']}\n"
+                "Template:\n"
+                f"{first['question_template']}\n"
+                "Your Code:\n"
+                f"{first['response']}\n\n"
+                "Example 2:\n"
+                f"Question: {second['question_name']} — {second['question_text']}\n"
+                "Template:\n"
+                f"{second['question_template']}\n"
+                "Your Code:\n"
+                f"{second['response']}\n\n"
+                "Example 3:\n"
+                f"Question: {third['question_name']} — {third['question_text']}\n"
+                "Template:\n"
+                f"{third['question_template']}\n"
+                "Your Code:\n"
+                f"{third['response']}\n\n"
+                "Now, using that same student's coding style, attempt this:\n"
+                "Ensure that the code works perfectly, but its efficiency should be based on students' past examples.\n"
+                "If a student has a tendency to write correct but inefficient code, imitate the inefficiency "
+                "but if they write efficiently, write efficiently too.\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code is correct, includes any class definition when needed, and handles all edge cases properly. "
+                "Return the code in C++ code block format, and nothing else, and produce only one set of code."
+            )
+
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(target_question_id),
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+
+        # Print summary statistics
+        print("\n=== INSTANCE CREATION SUMMARY ===")
+        print(f"Total instances created: {len(instances)}")
+        print(f"Skipped (insufficient data): {skipped_insufficient_data}")
+        print(f"Skipped (no test cases): {skipped_no_tests}")
+        print(f"Available test case question IDs: {len(available_question_ids)}")
+
+        if instances:
+            print("Sample created instances:")
+            for i, inst in enumerate(instances[:5]):
+                if inst.extra_data is None:
+                    test_count = 0
+                else:
+                    test_count = len(inst.extra_data.get("test_cases", []))
+                print(f"  {inst.id}: {test_count} test cases")
+
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}

helm/benchmark/scenarios/codeinsights_correct_code_scenario.py

@@ -0,0 +1,78 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, VALID_SPLIT
+import pandas as pd
+
+
+class CodeInsightsCorrectCodeScenario(Scenario):
+    name = "codeinsights_correct_code"
+    description = "Generate correct response code for C++ programming questions"
+    tags = ["codeinsights", "c++", "correct_code"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario1_2_data.csv")
+
+        # Load test cases (unit tests)
+        instances = []
+        for question_id, question_df in df.groupby("question_unittest_id"):
+            target = question_df.iloc[0]
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            prompt = (
+                f"Question: {target['question_name']} — {target['question_text']}\n\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code is correct, efficient, includes any class definition when needed, and handles all edge cases properly. "
+                "Return the code in C++ code block format, and nothing else."
+            )
+            instances.append(
+                Instance(
+                    id=f"{question_id}",
+                    input=Input(text=prompt),
+                    references=[],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(question_id) if question_id else None,
+                        "question_name": target.get("question_name", ""),
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+        return instances

helm/benchmark/scenarios/codeinsights_edge_case_scenario.py

@@ -0,0 +1,192 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsEdgeCaseScenario(Scenario):
+    name = "codeinsights_edge_case"
+    description = "Evaluate alignment in edge case failure between LLM-generated code and student code"
+    tags = ["codeinsights", "c++", "edge_case"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario5_data.csv")
+
+        student_topic = pd.read_csv(
+            "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/student_performace_by_topic.csv"
+        )
+
+        # Load test cases (unit tests)
+        test_cases = self._load_test_cases()
+
+        # Get available question IDs with test cases
+        available_question_ids = set()
+        if test_cases:
+            available_question_ids = set(test_cases.keys())
+            print(f"Loaded test cases for {len(available_question_ids)} questions")
+        else:
+            print("WARNING: No test cases loaded!")
+            return []
+
+        instances = []
+        skipped_no_tests = 0
+        skipped_insufficient_data = 0
+
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values("timestamp")
+            target = student_df.iloc[0]
+
+            # Check if target question has test cases BEFORE processing
+            target_question_id = target.get("question_unittest_id", None)
+            if not target_question_id or str(target_question_id) not in available_question_ids:
+                skipped_no_tests += 1
+                print(f"SKIPPING Student {student_id}, Question {target_question_id}: No test cases available")
+                continue
+
+            # Get test cases for this question (we know they exist now)
+            target_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                target_test_cases.append(testcase)
+
+            # Verify test cases are not empty
+            if not tc_parsing_success:
+                skipped_no_tests += 1
+                print(f"SKIPPING Student {student_id}, Question {target_question_id}: Empty test cases")
+                continue
+
+            if len(target_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                target_test_cases = target_test_cases[: self.num_testcases]
+
+            # Get student pass pattern for the target question
+            student_correctness_pattern = target.get("pass", None)
+            if student_correctness_pattern is not None:
+                main_part = int(student_correctness_pattern)
+                # Convert each character to an int
+                student_correctness_list = [int(ch) for ch in str(main_part)]
+            else:
+                student_correctness_list = []
+
+            # Student specific topic performance in previous attempts
+            student_level_prompt = f"Student {student_id} has the following performance across topics:\n"
+            topic_performance = student_topic[student_topic["student_id"] == student_id]
+            for _, row in topic_performance.iterrows():
+                topic = row["topic"]
+                pass_rate = round(row["pass_rate"], 2)
+                perfect = round(row["perfect"], 2)
+
+                student_level_prompt += (
+                    f"- For topic '{topic}', the unit test pass rate is {pass_rate}, "
+                    f"and the rate of passing all unit tests is {perfect}.\n"
+                )
+
+            print(f"\n=== ACCEPTED INSTANCE: Student {student_id}, Question {target_question_id} ===")
+            print(f"Test cases loaded: {len(target_test_cases)}")
+            print(f"Student correctness pattern: {student_correctness_list}")
+            print(f"Question name: {target.get('question_name', 'MISSING')}")
+
+            prompt = (
+                "You are analyzing a student’s likely mistakes on an upcoming programming problem.\n"
+                "Your task: **predict exactly ONE unit-test index (0-based) that the student is most likely to fail.**\n"  # noqa: E501
+                "Return *only* that integer. No explanation.\n\n"
+                "=== Student Profile ===\n"
+                f"{student_level_prompt}\n"
+                "For the given programming question, identify which unit test the student would fail considering "
+                "their past performance, as well as consideration of unit test difficulty.\n"
+                f"Week: {target['week']}\n"
+                f"Topic: {target['topic']}\n\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n"
+                f"Unit Tests: {target_test_cases}\n"
+                if target_test_cases
+                else ""
+                "Think silently about:\n"
+                "• Which test seems hardest for the given topic?\n"
+                "• Where has the student historically struggled?\n"
+                "• Any edge-cases in the tests’ inputs/outputs?\n\n"
+                "******** ANSWER FORMAT (STRICT) ********\n"
+                "<integer>\n"
+                "****************************************"
+            )
+
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": target_test_cases,
+                        "question_id": str(target_question_id),
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+
+        # Print summary statistics
+        print("\n=== INSTANCE CREATION SUMMARY ===")
+        print(f"Skipped (insufficient data): {skipped_insufficient_data}")
+        print(f"Skipped (no test cases): {skipped_no_tests}")
+        print(f"Available test case question IDs: {len(available_question_ids)}")
+
+        if len(instances) >= 5:
+            print("Sample created instances:")
+            for i, inst in enumerate(instances[:5]):
+                if inst.extra_data is None:
+                    test_count = 0
+                else:
+                    test_count = len(inst.extra_data.get("test_cases", []))
+                print(f"  {inst.id}: {test_count} test cases")
+
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}