crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/scenarios/codeinsights_student_coding_scenario.py

@@ -0,0 +1,162 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsStudentCodingScenario(Scenario):
+    name = "codeinsights_student_coding"
+    description = "Mimic student C++ style on foundational questions"
+    tags = ["codeinsights", "c++", "student_coding"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario1_2_data.csv")
+        student_topic = pd.read_csv(
+            "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/student_performace_by_topic.csv"
+        )
+
+        instances = []
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values("timestamp")
+            if len(student_df) < 4:
+                continue
+            first = student_df.iloc[0]
+            second = student_df.iloc[1]
+            third = student_df.iloc[2]
+            target = student_df.iloc[3]
+
+            # Get test cases for this question
+            question_id = target.get("question_unittest_id", None)
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            # Get student pass (0 or 1) for the target question
+            student_correctness_pattern = target.get("pass", None)
+            main_part = int(student_correctness_pattern)  # "1111111111"
+            # Convert each character to an int
+            student_correctness_list = [int(ch) for ch in str(main_part)]  # [1,1,1,1,1,1,1,1,1,1]
+
+            # Student specific topic performance in previous attempts
+            student_level_prompt = f"Student {student_id} has the following performance across topics:\n"
+            topic_performance = student_topic[student_topic["student_id"] == student_id]
+            for _, row in topic_performance.iterrows():
+                topic = row["topic"]
+                pass_rate = round(row["pass_rate"], 2)
+                perfect = round(row["perfect"], 2)
+
+                student_level_prompt += (
+                    f"- For topic '{topic}', the unit test pass rate is {pass_rate}, "
+                    f"and the rate of passing all unit tests is {perfect}.\n"
+                )
+
+            prompt = (
+                "=== Student Profile ===\n"
+                f"{student_level_prompt}\n"
+                f"Week: {target['week']}\n"
+                f"Topic: {target['topic']}\n\n"
+                "Example 1:\n"
+                f"Question: {first['question_name']} — {first['question_text']}\n"
+                "Template:\n"
+                f"{first['question_template']}\n"
+                "Your Code:\n"
+                f"{first['response']}\n\n"
+                "Example 2:\n"
+                f"Question: {second['question_name']} — {second['question_text']}\n"
+                "Template:\n"
+                f"{second['question_template']}\n"
+                "Your Code:\n"
+                f"{second['response']}\n\n"
+                "Example 3:\n"
+                f"Question: {third['question_name']} — {third['question_text']}\n"
+                "Template:\n"
+                f"{third['question_template']}\n"
+                "Your Code:\n"
+                f"{third['response']}\n\n"
+                "Now, using that same student style, attempt this:\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code includes any class definition when needed. "
+                "Return the code in C++ code block format, and nothing else."
+            )
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(question_id) if question_id else None,
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}

helm/benchmark/scenarios/codeinsights_student_mistake_scenario.py

@@ -0,0 +1,188 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsStudentMistakeScenario(Scenario):
+    name = "codeinsights_student_mistake"
+    description = "Mimic how students mistake their C++ codes on foundational questions"
+    tags = ["codeinsights", "c++", "student_mistake"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario3_data.csv")
+        student_topic = pd.read_csv(
+            "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/student_performace_by_topic.csv"
+        )
+
+        instances = []
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values(by=["student_id", "question_unittest_id", "timestamp"])
+            if len(student_df) < 4:
+                continue
+            first = student_df.iloc[0]
+            second = student_df.iloc[1]
+            third = student_df.iloc[2]
+            target = student_df.iloc[3]
+
+            # Get test cases for this question
+            question_id = target.get("question_unittest_id", None)
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            # Get student pass (0 or 1) for the target question
+            student_correctness_pattern = target.get("pass", None)
+            main_part = int(student_correctness_pattern)  # "1111111111"
+            # Convert each character to an int
+            student_correctness_list = [int(ch) for ch in str(main_part)]  # [1,1,1,1,1,1,1,1,1,1]
+
+            # Student specific topic performance in previous attempts
+            student_level_prompt = f"Student {student_id} has the following performance across topics:\n"
+            topic_performance = student_topic[student_topic["student_id"] == student_id]
+            for _, row in topic_performance.iterrows():
+                topic = row["topic"]
+                pass_rate = round(row["pass_rate"], 2)
+                perfect = round(row["perfect"], 2)
+
+                student_level_prompt += (
+                    f"- For topic '{topic}', the unit test pass rate is {pass_rate}, "
+                    f"and the rate of passing all unit tests is {perfect}.\n"
+                )
+
+            prompt = (
+                "=== Student Profile ===\n"
+                f"{student_level_prompt}\n"
+                "When students submit a code to the platform, it will be tested by number of unit tests, where"
+                "- Unit test pass rate = proportion of unit tests passed with the code \n"
+                "- Full pass rate   = proportion of code passing all unit tests\n\n"
+                "=== Past Mistake Examples ===\n"
+                "Example 1 (Week {first['week']}, Topic: {first['topic']}):\n"
+                f"Question: {first['question_name']} — {first['question_text']}\n"
+                "Template:\n"
+                f"{first['question_template']}\n"
+                "Student's Response Code with Error:\n"
+                f"{first['response_mistake']}\n\n"
+                "Example 2 (Week {second['week']}, Topic: {second['topic']}):\n"
+                f"Question: {second['question_name']} — {second['question_text']}\n"
+                "Template:\n"
+                f"{second['question_template']}\n"
+                "Student's Response Code with Error:\n"
+                f"{second['response_mistake']}\n\n"
+                "Example 3 (Week {third['week']}, Topic: {third['topic']}):\n"
+                f"Question: {third['question_name']} — {third['question_text']}\n"
+                "Template:\n"
+                f"{third['question_template']}\n"
+                "Student's Response Code with Error:\n"
+                f"{third['response_mistake']}\n\n"
+                "=== New Target Problem ===\n"
+                f"Week: {target['week']}, Topic: {target['topic']}\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "⚠**Instructions:**\n"
+                "1. Mimic your own coding style, naming conventions, indentation, and typical error patterns.\n"
+                "2. Introduce mistake you are likely to make (e.g., off‐by‐one index, wrong initialization, "
+                "missing edge case).\n"
+                "3. Do **not** produce a fully correct solution or add unfamiliar optimizations.\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code is includes any class definition when needed. "
+                "Return the code in C++ code block format, and nothing else."
+            )
+
+            print(f"\n=== DEBUG INFO FOR STUDENT {student_id}, QUESTION {question_id} ===")
+            print(f"Test cases loaded: {len(question_test_cases)}")
+            print(f"Student correctness pattern: {student_correctness_list}")
+            print(f"Original pass field: {target.get('pass', 'MISSING')}")
+            print(f"Question template exists: {'question_template' in target}")
+            print(f"Question name: {target.get('question_name', 'MISSING')}")
+
+            # Also add this validation in your UnitTestAlignmentMetric evaluate_generation method:
+            def evaluate_generation(self, adapter_spec, request_state, metric_service, eval_cache_path):
+                print("\n=== UNIT TEST METRIC DEBUG ===")
+                print(f"Has extra_data: {hasattr(request_state.instance, 'extra_data')}")
+                if hasattr(request_state.instance, "extra_data"):
+                    extra_data = request_state.instance.extra_data
+                    print(f"Extra data keys: {list(extra_data.keys())}")
+                    print(f"Test cases: {len(extra_data.get('test_cases', []))}")
+                    print(f"Student pattern: {extra_data.get('student_correctness_pattern', 'MISSING')}")
+
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response_mistake"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(question_id) if question_id else None,
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}

helm/benchmark/scenarios/dischargeme_scenario.py

@@ -1,5 +1,5 @@
 from typing import List
-from helm.common.general import ensure_directory_exists
+from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -21,26 +21,34 @@ def file_preprocessing(data_path: str, task_objective: str) -> pd.DataFrame:
     data_path is directory that contains the downloaded files: '{base_dir}/physionet.org/'
     """
     # Load the first CSV file
-    df_diagnosis = pd.read_csv(
-        f"{data_path}/files/discharge-me/1.3/test_phase_1/diagnosis.csv.gz", compression="gzip", keep_default_na=False
+    diagnosis_path = f"{data_path}/files/discharge-me/1.3/test_phase_1/diagnosis.csv.gz"
+    check_file_exists(
+        diagnosis_path, msg=f"[DischargeMeScenario] Required diagnosis file not found: '{diagnosis_path}'"
     )
-    df_discharge = pd.read_csv(
-        f"{data_path}/files/discharge-me/1.3/test_phase_1/discharge.csv.gz", compression="gzip", keep_default_na=False
+    discharge_path = f"{data_path}/files/discharge-me/1.3/test_phase_1/discharge.csv.gz"
+    check_file_exists(
+        discharge_path, msg=f"[DischargeMeScenario] Required discharge file not found: '{discharge_path}'"
     )
+    target_path = f"{data_path}/files/discharge-me/1.3/test_phase_1/discharge_target.csv.gz"
+    check_file_exists(target_path, msg=f"[DischargeMeScenario] Required target file not found: '{target_path}'")
+    radiology_path = f"{data_path}/files/discharge-me/1.3/test_phase_1/radiology.csv.gz"
+    check_file_exists(
+        radiology_path, msg=f"[DischargeMeScenario] Required radiology file not found: '{radiology_path}'"
+    )
+    ed_path = f"{data_path}/files/discharge-me/1.3/test_phase_1/edstays.csv.gz"
+    check_file_exists(ed_path, msg=f"[DischargeMeScenario] Required ed file not found: '{ed_path}'")
+    triage_path = f"{data_path}/files/discharge-me/1.3/test_phase_1/triage.csv.gz"
+    check_file_exists(triage_path, msg=f"[DischargeMeScenario] Required triage file not found: '{triage_path}'")
+    df_diagnosis = pd.read_csv(diagnosis_path, compression="gzip", keep_default_na=False)
+    df_discharge = pd.read_csv(discharge_path, compression="gzip", keep_default_na=False)
     df_target = pd.read_csv(
-        f"{data_path}/files/discharge-me/1.3/test_phase_1/discharge_target.csv.gz",
+        target_path,
         compression="gzip",
         keep_default_na=False,
     )
-    df_radiology = pd.read_csv(
-        f"{data_path}/files/discharge-me/1.3/test_phase_1/radiology.csv.gz", compression="gzip", keep_default_na=False
-    )
-    df_ed = pd.read_csv(
-        f"{data_path}/files/discharge-me/1.3/test_phase_1/edstays.csv.gz", compression="gzip", keep_default_na=False
-    )
-    df_triage = pd.read_csv(
-        f"{data_path}/files/discharge-me/1.3/test_phase_1/triage.csv.gz", compression="gzip", keep_default_na=False
-    )
+    df_radiology = pd.read_csv(radiology_path, compression="gzip", keep_default_na=False)
+    df_ed = pd.read_csv(ed_path, compression="gzip", keep_default_na=False)
+    df_triage = pd.read_csv(triage_path, compression="gzip", keep_default_na=False)
     df_diagnosis_triage = pd.merge(
         df_diagnosis, df_triage, on="subject_id", how="inner", suffixes=("_df_diagnosis", "_df_triage")
     )
@@ -113,16 +121,23 @@ class DischargeMeScenario(Scenario):
     """
 
     name = "dischargeme"
-    description = "DischargeMe is a discharge instruction generation dataset and brief hospital course generation \
-    dataset collected from MIMIC-IV data, consindering only the discharge text as well as the radiology report text."
+    description = (
+        "DischargeMe is a benchmark designed to evaluate clinical text generation. It pairs"
+        "discharge summaries and radiology reports from MIMIC-IV with generation tasks"
+        "such as writing discharge instructions or summarizing the brief hospital course. The"
+        "benchmark assesses a model's ability to generate patient-facing documentation that is"
+        "complete, empathetic, and clinically accurate."
+    )
     tags = ["biomedical"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/share/pi/nigam/data/physionet.org"
-        ensure_directory_exists(data_path)
         instances: List[Instance] = []
-        df_bhc = file_preprocessing(data_path, "brief_hospital_course")
-        df_di = file_preprocessing(data_path, "discharge_instructions")
+        df_bhc = file_preprocessing(self.data_path, "brief_hospital_course")
+        df_di = file_preprocessing(self.data_path, "discharge_instructions")
 
         for i in range(df_bhc.shape[0]):
             prompt_bhc = create_prompt(

helm/benchmark/scenarios/ehr_sql_scenario.py

@@ -36,7 +36,13 @@ class EhrSqlScenario(Scenario):
     )
 
     name = "ehr_sql"
-    description = "Given a natural language instruction, generate an SQL query that would be used in clinical research."
+    description = (
+        "EHRSQL is a benchmark designed to evaluate models on generating structured queries"
+        "for clinical research. Each example includes a natural language question and a database"
+        "schema, and the task is to produce an SQL query that would return the correct result"
+        "for a biomedical research objective. This benchmark assesses a model's understanding"
+        "of medical terminology, data structures, and query construction."
+    )
     tags = ["sql", "medical", "reasoning"]
 
     def setup_database(self, output_path: str) -> str:

helm/benchmark/scenarios/ehrshot_scenario.py

@@ -3,12 +3,11 @@ import os
 import pandas as pd
 import tiktoken
 
-from filelock import FileLock
 from functools import partial
 from tqdm import tqdm
 from typing import Any, Dict, List, Optional, Mapping
 
-from helm.common.general import ensure_directory_exists
+from helm.common.general import check_file_exists, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
@@ -1411,7 +1410,10 @@ class EHRSHOTScenario(Scenario):
 
     name = "ehrshot"
     description = (
-        "A dataset given a patient record of EHR codes, classifying if an event will occur at a future date or not."
+        "EHRSHOT is a benchmark designed to evaluate a model's ability to predict future"
+        "clinical events using structured EHR data. Each instance contains a patient's"
+        "historical EHR data and a forward-looking clinical question about whether a particular"
+        "diagnosis, lab result, or hospital event will occur."
     )
     tags = []  # TODO
 
@@ -1420,24 +1422,32 @@ class EHRSHOTScenario(Scenario):
         "no",
     ]
 
-    def __init__(self, subject: str, max_length: Optional[int] = None):
+    def __init__(self, subject: str, data_path: str, max_length: Optional[int] = None):
         super().__init__()
         self.subject: str = subject  # same as "task" or "labeling_function"
-        self.path_to_meds_dir: str = "/share/pi/nigam/data/medhelm/ehrshot/meds/"
-        self.path_to_tmp_dir: str = "/share/pi/nigam/data/medhelm/ehrshot/prompts/"
         self.max_length = max_length
+        self.data_path = data_path
 
-    def create_benchmark(self, n_procs: int = 4) -> Dict[str, str]:
+    def create_benchmark(self, output_path: str, n_procs: int = 4) -> Dict[str, str]:
         """Loads the MEDS dataset and converts it to prompts"""
-
         # Load MEDS EHRSHOT patient timelines
-        df_data = pd.read_parquet(os.path.join(self.path_to_meds_dir, "data/data.parquet"))
-        df_splits = pd.read_parquet(os.path.join(self.path_to_meds_dir, "metadata/subject_splits.parquet"))
-
+        data_parquet_path = os.path.join(self.data_path, "data/data.parquet")
+        check_file_exists(
+            data_parquet_path, msg=f"[EHRSHOTScenario] Required parquet data file not found: '{data_parquet_path}'"
+        )
+        splits_parquet_path = os.path.join(self.data_path, "metadata/subject_splits.parquet")
+        check_file_exists(
+            splits_parquet_path, msg=f"[EHRSHOTScenario] Required splits file not found: '{splits_parquet_path}'"
+        )
+        df_data = pd.read_parquet(data_parquet_path)
+        df_splits = pd.read_parquet(splits_parquet_path)
         # Load MEDS EHRSHOT labels
-        tasks = sorted(os.listdir(os.path.join(self.path_to_meds_dir, "labels")))
+        tasks = sorted(os.listdir(os.path.join(self.data_path, "labels")))
         for t in tasks:
-            path_to_labels: str = os.path.join(self.path_to_meds_dir, "labels", t, "labels.parquet")
+            path_to_labels: str = os.path.join(self.data_path, "labels", t, "labels.parquet")
+            check_file_exists(
+                path_to_labels, msg=f"[EHRSHOTScenario] Required labels file not found: '{path_to_labels}'"
+            )
             if t != self.subject or not os.path.exists(path_to_labels):
                 continue
             df_labels = pd.read_parquet(path_to_labels)
@@ -1470,18 +1480,16 @@ class EHRSHOTScenario(Scenario):
         df_labels["prompt"] = prompts
 
         # Save to parquet
-        path_to_output_dir: str = os.path.join(self.path_to_tmp_dir, self.subject)
+        path_to_output_dir: str = os.path.join(output_path, self.subject)
         ensure_directory_exists(path_to_output_dir)
         df_labels.to_parquet(os.path.join(path_to_output_dir, "medhelm_prompts.parquet"))
         return {"status": "success"}
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        path_to_input_csv: str = os.path.join(self.path_to_tmp_dir, self.subject, "medhelm_prompts.parquet")
-        lock_path = path_to_input_csv + ".lock"
-        with FileLock(lock_path):
-            if not os.path.exists(path_to_input_csv):
-                print(f"Creating benchmark from SCRATCH for {self.subject}...")
-                self.create_benchmark()  # Create benchmark from scratch
+        path_to_input_csv: str = os.path.join(output_path, self.subject, "medhelm_prompts.parquet")
+        if not os.path.exists(path_to_input_csv):
+            print(f"Creating benchmark from SCRATCH for {self.subject}...")
+            self.create_benchmark(output_path=output_path)  # Create benchmark from scratch
 
         # Load data for this task
         df = pd.read_parquet(path_to_input_csv)
@@ -1509,38 +1517,3 @@ class EHRSHOTScenario(Scenario):
             )
 
         return instances
-
-
-if __name__ == "__main__":
-    # Generate statistics on prompts
-    from transformers import AutoTokenizer
-
-    tokenizer = AutoTokenizer.from_pretrained("gpt2")
-    tqdm.pandas()
-    n_procs: int = 10
-
-    os.makedirs("./ehrshot_stats", exist_ok=True)
-    for t in TASK_FULL_NAMES.keys():
-        # Skip if already exists
-        if os.path.exists(f"./ehrshot_stats/{t}.txt"):
-            print(f"Skipping {t} because it already exists")
-            continue
-
-        # Create benchmark
-        scenario = EHRSHOTScenario(subject=t)
-        scenario.create_benchmark(n_procs=n_procs)
-        instances = scenario.get_instances("test.csv")
-
-        # Calculate prompt token stats
-        path_to_input_csv = os.path.join(scenario.path_to_tmp_dir, scenario.subject, "medhelm_prompts.parquet")
-        df = pd.read_parquet(path_to_input_csv)
-        df["prompt_n_tokens"] = df["prompt"].progress_apply(lambda x: len(tokenizer.encode(x)))
-        with open(f"./ehrshot_stats/{t}.txt", "w") as f:
-            f.write("-" * 100 + "\n")
-            f.write(f"Task: {t}\n")
-            f.write(f"# of instances: {len(instances)}\n")
-            f.write(f"# of positives: {df['boolean_value'].sum()}\n")
-            f.write(f"Size of splits:\n{df['split'].value_counts()}\n")
-            f.write(f"# tokens per prompt:\n{df['prompt_n_tokens'].describe()}\n")
-            f.write("-" * 100 + "\n")
-        df.to_parquet(os.path.join(scenario.path_to_tmp_dir, scenario.subject, "medhelm_prompts.parquet"))