crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/scenarios/lmkt_scenarios.py

@@ -0,0 +1,288 @@
+"""Cultural alignment evaluation scenario based on Vietnam World Values Survey responses."""
+
+import os
+import json
+import random
+from typing import List
+from datasets import load_dataset
+from huggingface_hub import snapshot_download
+
+from helm.common.hierarchical_logger import hlog, hwarn
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+SUPPORTED_LANGUAGES = ["en", "vi"]
+
+
+class CulturalValueUnderstandingWVSScenario(Scenario):
+    """Cultural values understanding evaluation based on Vietnam World Values Survey responses."""
+
+    name = "cultural_value_understanding_wvs"
+    description = "Evaluates model understanding of cultural values from WVS Wave 7"
+    tags = ["cultural_value_understanding"]
+
+    def __init__(
+        self,
+        language: str,
+        personas_filename: str = "vn_personas_with_filtered_responses.json",
+        questions_filename: str = "wvs_questions_dump.en.json",
+        repo_id: str = "stair-lab/cultural-alignment-dataset",
+        num_personas: int = 10,
+        num_question_variants: int = 2,
+        include_few_shot_examples: bool = False,
+    ):
+        super().__init__()
+        self.language = language
+        self.personas_filename = personas_filename
+        self.questions_filename = questions_filename
+        self.repo_id = repo_id
+        self.num_personas = 10 if isinstance(num_personas, str) else num_personas
+        self.num_question_variants = min(num_question_variants, 4)
+        self.include_few_shot_examples = include_few_shot_examples
+
+        if self.language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {self.language}. Supported languages are: {SUPPORTED_LANGUAGES}")
+
+    def get_few_shot_examples(self) -> str:
+        """Return a few-shot learning examples formatted as a string."""
+        examples = """Here are some examples:
+
+Example 1:
+You are a 35 year old male from Vietnam. You have higher education and consider yourself middle class. You are married.
+
+How important is family in your life?
+
+1. Very important
+2. Rather important
+3. Not very important
+4. Not at all important
+
+Please select the option number that best represents your view.
+1
+
+Example 2:
+You are a 28 year old female from Vietnam. You have middle education and consider yourself working class. You are single.
+
+Would you say that most people can be trusted or that you need to be very careful in dealing with people?
+
+1. Most people can be trusted
+2. Need to be very careful
+
+Please select the option number that best represents your view.
+2
+
+Example 3:
+You are a 50 year old male from Vietnam. You have lower education and consider yourself lower class. You are married.
+
+Do you think that homosexuality is justifiable?
+
+1. Never justifiable
+2. Rarely justifiable
+3. Sometimes justifiable
+4. Always justifiable
+
+Please select the option number that best represents your view.
+1
+
+Now answer the following question:
+"""  # noqa: E501
+        return examples
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Generate test instances from Vietnam personas and WVS questions."""
+        instances: List[Instance] = []
+
+        # Download files from Hugging Face Hub
+        repo_local_path = snapshot_download(
+            repo_id=self.repo_id, repo_type="dataset", revision="fe54b6f5d75cfca5377707cd7199e39f517e3a1f"
+        )
+
+        # Load the downloaded files
+        with open(os.path.join(repo_local_path, self.personas_filename), "r", encoding="utf-8") as f:
+            personas = json.load(f)
+
+        with open(os.path.join(repo_local_path, self.questions_filename), "r", encoding="utf-8") as f:
+            questions = json.load(f)
+
+        # Get few-shot examples
+        few_shot_examples = self.get_few_shot_examples() if self.include_few_shot_examples else ""
+
+        # Sample personas
+        sampled_personas = random.sample(personas, min(self.num_personas, len(personas)))
+
+        # Create instances for each persona and question
+        for persona in sampled_personas:
+            # Get demographic info for persona description
+            persona_desc = (
+                f"You are a {persona.get('age', 'adult')} year old {persona.get('sex', 'person')} from Vietnam. "
+            )
+            persona_desc += f"You have {persona.get('education', 'some')} education and consider yourself {persona.get('social_class', 'middle class')}. "  # noqa: E501
+            persona_desc += f"You are {persona.get('marital_status', 'single')}."
+
+            # Process each question this persona answered
+            for qid, human_response in persona.get("responses", {}).items():
+                # Skip if no human response or if it's 0 (which might be a "Don't know" response)
+                if human_response is None:
+                    continue
+
+                # Convert human_response to int (if possible)
+                try:
+                    human_response_int = int(human_response)
+                except (ValueError, TypeError):
+                    # Skip if human_response can't be converted to int
+                    continue
+
+                # Get question info
+                question_data = questions.get(qid, {})
+                if not question_data:
+                    continue
+
+                # Get options directly from question data
+                q_options = question_data.get("options", [])
+                if not q_options:
+                    continue
+
+                # Skip if human_response is out of range
+                if human_response_int < 0 or human_response_int > len(q_options):
+                    continue
+
+                # Special handling for "Don't know" or zero responses
+                if human_response_int == 0:
+                    # Some questions might encode "Don't know" as 0
+                    # Skip for now, or you could add special handling
+                    continue
+
+                # Use the predefined question variations
+                question_variants = question_data.get("questions", [])
+                if not question_variants:
+                    question_variants = [f"Question {qid}: {question_data.get('description', '')}"]
+
+                # Use the specified number of variants
+                variants_to_use = min(self.num_question_variants, len(question_variants))
+                selected_variants = question_variants[:variants_to_use]
+
+                # Create instances for each selected question variant
+                for q_text in selected_variants:
+                    # Format the prompt with or without few-shot examples
+                    if self.include_few_shot_examples:
+                        prompt = f"{few_shot_examples}{persona_desc}\n\n{q_text}\n\n"
+                    else:
+                        prompt = f"{persona_desc}\n\n{q_text}\n\n"
+
+                    # Add options from question data - with numbers, not letters
+                    for i, opt in enumerate(q_options, 1):
+                        prompt += f"{i}. {opt}\n"
+
+                    prompt += "\nPlease select the option number that best represents your view. Return only the option number. Do not return anything else."  # noqa: E501
+
+                    # Create a reference with just the human response number
+                    # We don't create multiple references, just use the actual human response
+                    reference = Reference(Output(text=str(human_response_int)), tags=[CORRECT_TAG])
+
+                    # Create the instance
+                    instance = Instance(
+                        input=Input(text=prompt),
+                        references=[reference],
+                        split=TEST_SPLIT,
+                    )
+                    instances.append(instance)
+
+        if not instances:
+            hwarn("No valid instances were created. Check the input data and parameters.")
+        else:
+            hlog(f"Created {len(instances)} instances for cultural value understanding scenario.")
+
+        return instances
+
+
+class SocialNormApplicationNormADScenario(Scenario):
+    """Social norms application evaluation based on Vietnam World Values Survey responses."""
+
+    name = "social_norm_application_normad"
+    description = "Evaluates model understanding of social norms from WVS Wave 7"
+    tags = ["social_norm_application"]
+
+    def __init__(
+        self,
+        language: str,
+    ):
+        super().__init__()
+        self.language = language
+
+        if self.language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {self.language}. Supported languages are: {SUPPORTED_LANGUAGES}")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset = load_dataset(
+            "akhilayerukola/NormAd", split="train", revision="311049f5610e932792672e4e13e7df62b6290336"
+        )
+
+        instances: List[Instance] = []
+
+        for item in dataset:
+            input_text = (
+                f"Assumming that below norms and story are from {item['Country'].capitalize()}:\n"
+                f"{item['Background']}\n\n{item['Story']}\n"
+                "Return one of the following options: 'yes', 'no', or 'neutral'. Do not return anything else.\n\n"
+            )
+
+            instance = Instance(
+                input=Input(text=input_text),
+                references=[Reference(Output(text=item["Gold Label"]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances
+
+
+class SocialNormExplanationNormADScenario(Scenario):
+    """Social norms explanation evaluation based on Vietnam World Values Survey responses."""
+
+    name = "social_norm_explanation_normad"
+    description = "Evaluates model understanding of social norms from WVS Wave 7"
+    tags = ["social_norm_explanation"]
+
+    def __init__(
+        self,
+        language: str,
+    ):
+        super().__init__()
+        self.language = language
+
+        if self.language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {self.language}. Supported languages are: {SUPPORTED_LANGUAGES}")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset = load_dataset(
+            "akhilayerukola/NormAd", split="train", revision="311049f5610e932792672e4e13e7df62b6290336"
+        )
+
+        instances: List[Instance] = []
+
+        for item in dataset:
+            if not item["Explanation"]:
+                continue
+
+            input_text = (
+                f"Assumming that below norms and story are from {item['Country'].capitalize()}:\n"
+                f"{item['Background']}\n\n{item['Story']}\n"
+                f"The answer is {item['Gold Label']}. "
+                "Briefly explain the reasoning behind this answer in one or two sentences.\n\n"
+            )
+
+            instance = Instance(
+                input=Input(text=input_text),
+                references=[Reference(Output(text=item["Explanation"]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/math_scenario.py

@@ -18,13 +18,14 @@ from helm.benchmark.scenarios.scenario import (
 
 
 def remove_boxed(string: str) -> Optional[str]:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
-    Extract the text within a \\boxed{...} environment.
+    Extract the text within a \boxed{...} environment.
 
     Example:
-    >>> remove_boxed(\\boxed{\\frac{2}{3}})
-    \\frac{2}{3}
+        >>> from helm.benchmark.scenarios.math_scenario import *  # NOQA
+        >>> remove_boxed(r'\boxed{\frac{2}{3}}')
+        '\\frac{2}{3}'
     """
     left = "\\boxed{"
     try:
@@ -68,17 +69,17 @@ def last_boxed_only_string(string: str) -> Optional[str]:
 
 
 def _fix_fracs(string: str) -> str:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
     Reformat fractions.
 
     Examples:
-    >>> _fix_fracs("\\frac1b")
-    \frac{1}{b}
-    >>> _fix_fracs("\\frac12")
-    \frac{1}{2}
-    >>> _fix_fracs("\\frac1{72}")
-    \frac{1}{72}
+        >>> _fix_fracs(r"\frac1b")
+        '\\frac{1}{b}'
+        >>> _fix_fracs(r"\frac12")
+        '\\frac{1}{2}'
+        >>> _fix_fracs(r"\frac1{72}")
+        '\\frac{1}{72}'
     """
     substrs = string.split("\\frac")
     new_str = substrs[0]
@@ -112,13 +113,13 @@ def _fix_fracs(string: str) -> str:
 
 
 def _fix_a_slash_b(string: str) -> str:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
     Reformat fractions formatted as a/b to \\frac{a}{b}.
 
     Example:
-    >>> _fix_a_slash_b("2/3")
-    \frac{2}{3}
+        >>> _fix_a_slash_b(r"2/3")
+        '\\frac{2}{3}'
     """
     if len(string.split("/")) != 2:
         return string
@@ -149,13 +150,13 @@ def _remove_right_units(string: str) -> str:
 
 
 def _fix_sqrt(string: str) -> str:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
     Reformat square roots.
 
     Example:
-    >>> _fix_sqrt("\\sqrt3")
-    \sqrt{3}
+        >>> _fix_sqrt("\\sqrt3")
+        '\\sqrt{3}'
     """
     if "\\sqrt" not in string:
         return string
@@ -210,7 +211,7 @@ def _strip_string(string: str) -> str:
 
     # remove percentage
     string = string.replace("\\%", "")
-    string = string.replace("\%", "")
+    string = string.replace(r"\%", "")
 
     # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
     string = string.replace(" .", " 0.")
@@ -391,13 +392,13 @@ class MATHScenario(Scenario):
         for split, split_name in zip([TRAIN_SPLIT, TEST_SPLIT], ["train", "test"]):
             if split == TRAIN_SPLIT and self.use_official_examples:
                 train_instances = [
-                    ("What is $\left(\\frac{7}{8}\\right)^3 \cdot \left(\\frac{7}{8}\\right)^{-3}$?", "1"),
+                    ("What is $\\left(\\frac{7}{8}\\right)^3 \\cdot \\left(\\frac{7}{8}\\right)^{-3}$?", "1"),
                     (
                         "In how many ways can 4 books be selected from a shelf of 6 books"
                         + " if the order in which the books are selected does not matter?",
                         "15",
                     ),
-                    ("Find the distance between the points $(2,1,-4)$ and $(5,8,-3).$", "\sqrt{59}"),
+                    ("Find the distance between the points $(2,1,-4)$ and $(5,8,-3).$", "\\sqrt{59}"),
                     (
                         "The faces of an octahedral die are labeled with digits $1$ through $8$."
                         + " What is the probability, expressed as a common fraction,"

helm/benchmark/scenarios/med_dialog_scenario.py

@@ -90,7 +90,12 @@ class MedDialogScenario(Scenario):
     """
 
     name = "med_dialog"
-    description = "A collection of doctor-patient conversations with corresponding summaries."
+    description = (
+        "MedDialog is a benchmark of real-world doctor-patient conversations focused on health-related"
+        "concerns and advice. Each dialogue is paired with a one-sentence summary"
+        "that reflects the core patient question or exchange. The benchmark evaluates a model's"
+        "ability to condense medical dialogue into concise, informative summaries."
+    )
     tags = ["dialogue", "biomedical"]
 
     def __init__(self, subset: str):

helm/benchmark/scenarios/medalign_scenario.py

@@ -60,12 +60,18 @@ class MedalignScenario(Scenario):
     """
 
     name = "medalign"
-    description = "A dataset that asks models to answer questions/follow instructions over longitudinal EHR."
+    description = (
+        "MedAlign is a benchmark that evaluates a model's ability to interpret and follow"
+        "instructions grounded in longitudinal electronic health records (EHR). Each instance"
+        "includes an event-stream style patient record and a natural language question or task,"
+        "requiring clinically informed reading comprehension and reasoning."
+    )
     tags = ["knowledge", "reasoning", "biomedical"]
 
-    def __init__(self, max_length: int):
+    def __init__(self, max_length: int, data_path: str):
         super().__init__()
         self.max_length = max_length
+        self.data_path = data_path
 
     def process_tsv(self, data) -> List[Instance]:
         instances: List[Instance] = []
@@ -84,5 +90,5 @@ class MedalignScenario(Scenario):
         return instances
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        dataset = return_dataset_dataframe(self.max_length)
+        dataset = return_dataset_dataframe(self.max_length, self.data_path)
         return self.process_tsv(dataset)

helm/benchmark/scenarios/medalign_scenario_helper.py

@@ -2,23 +2,15 @@
 # type: ignore
 # fmt: off
 
-import ast
-import datetime
 import transformers
-import langchain
-import langchain.prompts
-import lxml.etree
 import os
 import pandas as pd
-import re
 import tiktoken
 
-from langchain_community.retrievers import BM25Retriever
 from tqdm import tqdm
-from typing import Any, Dict, Optional, Union, Callable
-from langchain.schema import Document
-import langchain_community
+from typing import Any, Dict, Optional, Callable
 
+from helm.common.general import check_file_exists
 
 
 def get_instructions(path_to_instructions: str) -> Dict[int, Dict[str, Any]]:
@@ -166,102 +158,13 @@ def get_tokenizer(tokenizer_name: str) -> Callable:
     return transformers.AutoTokenizer.from_pretrained(tokenizer_name, legacy=False)
 
 
-def retrieve_most_relevant_visits(ehr_visit_strs, query, target_length, tokenizer):
-    """
-    Retrieve and filter relevant EHR visits based on a query and target length.
-
-    This function retrieves electronic health record (EHR) visit strings, sorts them
-    by relevance using the BM25Retriever, and constructs a list of final documents
-    that fit within a specified character length. The final list ensures that the
-    most important visit isn't cut off and is sorted chronologically.
-
-    Parameters:
-        ehr_visit_strs (list of str): List of EHR visit strings.
-        query (str): Query string to retrieve relevant visits.
-        target_length (int): Maximum total token count for the final list of documents.
-        tokenizer (Callable): Tokenizer that converts text to tokens (used for tracking context length)
-
-    Returns:
-        list[str]: List of EHR visit strings sorted chronologically and constrained by the target length.
-    """
-    ehr_visits=re.split(r'(?=</encounter>\n)',ehr_visit_strs)
-    langchain_docs = [
-        langchain.schema.Document(page_content=doc) for doc in ehr_visits #broken since ehr_visit_strs is one string of all visits
-    ]
-    # `k` is the number of documents to retrieve
-    # We retrieve everything and just use the BM25Retriever to sort the documents
-    retriever = langchain_community.retrievers.BM25Retriever.from_documents(
-        langchain_docs, k=len(langchain_docs)
-    )
-
-    # Invoking the retriever means the most relevant documents are sorted first
-    sorted_docs = retriever.invoke(query)
-
-    # Define the regex pattern to find the start time
-    # pattern = r'start="([\d/]+ [\d:]+)"'
-    pattern = r'start="([\d/]+ [\d:]+ ?[APM]{0,2})"'
-
-    docs = []
-    dts = []
-
-    # Find the startime of the document
-    for doc in sorted_docs:
-        doc_content = doc.page_content
-        start_dt_match = re.search(pattern, doc_content)
-        if start_dt_match:
-            start_dt = start_dt_match.group(1)
-            parsed = False
-            # Try different date formats
-            for fmt in (
-                "%m/%d/%y %I:%M %p",
-                "%m/%d/%Y %I:%M %p",
-                "%m/%d/%y %H:%M",
-                "%m/%d/%Y %H:%M",
-            ):
-                try:
-                    dts.append(datetime.datetime.strptime(start_dt, fmt))
-                    parsed = True
-                    break
-                except ValueError:
-                    continue
-            if not parsed:
-                print(f"Error parsing date: {start_dt}")
-                continue
-        else:
-            print(f"Start time not found., {doc_content}")
-            dts.append(datetime.datetime.min)
-        docs.append(doc_content)
-
-    final_docs = []
-    current_length = 0
-
-    # Add documents until we exceed the allocated context length
-    for i in range(len(docs)):
-        doc_content = docs[i]
-        doc_length = len(tokenizer.encode(doc_content))
-        final_docs.append((dts[i], doc_content))
-        current_length += doc_length
-        if current_length > target_length:
-            break
-
-    # Sort final_docs chronologically
-    final_docs.sort(key=lambda x: x[0])
-
-    # Extract only the document content for the final output
-    final_docs_content = [doc_content for _, doc_content in final_docs]
-
-    return final_docs_content
-
-
-
 def pack_and_trim_prompts(
     instructions: Dict[int, Dict[str, str]],
     ehrs: Dict[int, str],
-    prompt_template: langchain.prompts.PromptTemplate,
+    prompt_string: str,
     context_length: int,
     generation_length: int,
     tokenizer: Any,
-    use_RAG: bool = True,
     verbose: bool = False,
     include_ehr: bool = True,
 ) -> Dict[int, str]:
@@ -275,26 +178,15 @@ def pack_and_trim_prompts(
         patient_id = int(instructions[instruction_id]["patient_id"])
         relevant_ehr = ehrs[patient_id]
 
-        # Calculate how many tokens of EHR we can include in the prompt
         num_tokens_instruction = len(tokenizer.encode(instruction))
-        num_tokens_prompt_template = len(tokenizer.encode(prompt_template.template))
+        num_tokens_prompt_template = len(tokenizer.encode(prompt_string))
         if include_ehr:
             target_ehr_length = context_length - generation_length - num_tokens_prompt_template - num_tokens_instruction
         else:
             target_ehr_length = 0
         if target_ehr_length <= 0:
-            prompt_with_truncated_ehr = prompt_template.format(question=instruction, ehr="")
+            prompt_with_truncated_ehr = prompt_string.format(question=instruction, ehr="")
         else:
-            if use_RAG:
-                # Return a list of the most relevant visit strings
-                most_relevant_visits = retrieve_most_relevant_visits(
-                    ehr_visit_strs=relevant_ehr,
-                    query=instruction,
-                    target_length=target_ehr_length,
-                    tokenizer=tokenizer,
-                )
-                relevant_ehr = "\n".join(most_relevant_visits)
-
             # Do a first pass with a fast tokenizer
             fast_tokenizer = tiktoken.get_encoding("cl100k_base")
             fast_encoded = fast_tokenizer.encode(relevant_ehr)
@@ -306,13 +198,17 @@ def pack_and_trim_prompts(
                 encoded_ehr = tokenizer.encode(fast_truncated_ehr)
                 truncated_encoded_ehr = encoded_ehr[-target_ehr_length:]
                 truncated_ehr = tokenizer.decode(truncated_encoded_ehr)
-                prompt_with_truncated_ehr = prompt_template.format(question=instruction, ehr=truncated_ehr)
+                prompt_with_truncated_ehr = prompt_string.format(question=instruction, ehr=truncated_ehr)
+            else:
+                # If the fast encoding is still too long, just use the full EHR up to allowed length
+                truncated_ehr = fast_tokenizer.decode(fast_encoded[-target_ehr_length:])
+                prompt_with_truncated_ehr = prompt_string.format(question=instruction, ehr=truncated_ehr)
 
-                prompts_map[instruction_id] = prompt_with_truncated_ehr
+        prompts_map[instruction_id] = prompt_with_truncated_ehr
 
-                if verbose:
-                    print(prompt_with_truncated_ehr)
-                    print("~" * 20)
+        if verbose:
+            print(prompt_with_truncated_ehr)
+            print("~" * 20)
     return prompts_map
 
 
@@ -321,7 +217,6 @@ def preprocess_prompts(
     generation_length,
     path_to_instructions,
     path_to_ehrs,
-    use_RAG,
     include_ehr,
     tokenizer,
     codes_only=False,
@@ -346,16 +241,18 @@ def preprocess_prompts(
 
     # CONSTRUCT & TRUNCATE PROMPTS #
     print("Constructing prompts using instructions and EHRs...")
-    prompt_string="Instruction: Answer the following question based on the EHR:\n\nEHR: {ehr}\n\nQuestion: {question}\n\nAnswer:"
-    prompt_template = langchain.prompts.PromptTemplate.from_template(prompt_string)
+    prompt_string = (
+        "Instruction: Answer the following question based on the EHR:\n\n"
+        "EHR: {ehr}\n\nQuestion: {question}\n\nAnswer:"
+    )
+
     filled_prompts = pack_and_trim_prompts(
         instructions=instructions,
         ehrs=ehrs,
-        prompt_template=prompt_template,
+        prompt_string=prompt_string,
         context_length=target_context_length,
         generation_length=generation_length,
         tokenizer=tokenizer,
-        use_RAG=use_RAG,
         verbose=False,
         include_ehr=include_ehr,
     )
@@ -399,20 +296,21 @@ def add_reference_responses(prompts_df, path_to_reference_responses) -> pd.DataF
     Returns:
     pd.DataFrame: DataFrame containing the processed data.
     """
-    gold_df = pd.read_csv(path_to_reference_responses)
+    gold_df = pd.read_csv(path_to_reference_responses, sep='\t')
     gold_df = gold_df.query("annotator_num == 'Annotator_1'")
     gold_df = gold_df[["instruction_id", "clinician_response"]]
     merged_df = gold_df.merge(prompts_df, on="instruction_id", how="inner")
     return merged_df
 
 
-def return_dataset_dataframe(max_length: int) -> pd.DataFrame:
+def return_dataset_dataframe(max_length: int, data_path: str) -> pd.DataFrame:
     target_context_length = max_length
     generation_length = 256
-    path_to_instructions = "/share/pi/nigam/datasets/medalign_release_fixes/clinician-reviewed-model-responses.tsv"
-    path_to_ehrs = "/share/pi/nigam/datasets/medalign_release_fixes/medalign_ehr_xml"
-    path_to_reference_responses = "/share/pi/nigam/scottyf/clinician-instruction-responses.csv"
-    use_RAG = False
+    path_to_instructions = os.path.join(data_path, "clinician-reviewed-model-responses.tsv")
+    check_file_exists(path_to_instructions, msg=f"[MedAlignScenario] Required instructions file not found: '{path_to_instructions}'")
+    path_to_ehrs = os.path.join(data_path, "medalign_ehr_xml")
+    path_to_reference_responses = os.path.join(data_path, "clinician-instruction-responses.tsv")
+    check_file_exists(path_to_reference_responses, msg=f"[MedAlignScenario] Required clinician responses file not found: '{path_to_reference_responses}'")
     include_ehr = True
     tokenizer = "tiktoken"
 
@@ -421,7 +319,6 @@ def return_dataset_dataframe(max_length: int) -> pd.DataFrame:
         generation_length=generation_length,
         path_to_instructions=path_to_instructions,
         path_to_ehrs=path_to_ehrs,
-        use_RAG=use_RAG,
         include_ehr=include_ehr,
         tokenizer=tokenizer,
     )