crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/scenarios/casehold_scenario.py

@@ -0,0 +1,79 @@
+from typing import List
+import os
+import os.path
+
+from datasets import load_dataset, DatasetDict
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Output,
+)
+
+
+class CaseHOLDScenario(Scenario):
+    """
+     CaseHOLD QA
+       CaseHOLD is a multiple choice question answering task derived from legal citations in judicial rulings.
+       CaseHOLD consists of ~53,000 questions, mined from the Harvard Law Library case law corpus.
+
+     Dataset repository
+       https://huggingface.co/datasets/casehold/casehold
+     Publication
+       "When Does Pretraining Help? Assessing Self-Supervised Learning for Law and the CaseHOLD Dataset"
+       ICAIL, 2021
+       https://reglab.stanford.edu/data/casehold-benchmark/
+       https://arxiv.org/abs/2104.08671
+
+    Data content
+      The citing context from the judicial decision serves as the prompt for the question.
+      The answer choices are holding statements derived from citations following text in a legal decision.
+      There are five answer choices for each citing text.
+      The correct answer is the holding statement that corresponds to the citing text.
+      The four incorrect answers are other holding statements.
+
+    """  # noqa: E501
+
+    name = "casehold"
+    description = "CaseHOLD (Case Holdings On Legal Decisions) is a multiple choice question answering scenario where the task is to identify the relevant holding of a cited case [(Zheng et al, 2021)](https://arxiv.org/pdf/2104.08671.pdf)."  # noqa: E501
+    tags = ["question_answering", "legal"]
+
+    # Note: Skip the validation split since we don't need it
+    HELM_SPLIT_NAME_TO_DATASETS_SPLIT_NAME = {TRAIN_SPLIT: "train", TEST_SPLIT: "test"}
+    NUM_REFERENCES = 5
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path: str = os.path.join(output_path, "data")
+        ensure_directory_exists(data_path)
+        dataset: DatasetDict = load_dataset(
+            "casehold/casehold",
+            "all",
+            cache_dir=data_path,
+        )
+
+        instances: List[Instance] = []
+        for helm_split_name, datasets_split_name in self.HELM_SPLIT_NAME_TO_DATASETS_SPLIT_NAME.items():
+            split_data = dataset[datasets_split_name]
+            for example in split_data:
+                example_id = example["example_id"]
+                citing_prompt = example["citing_prompt"]
+                holdings = [example[f"holding_{i}"] for i in range(self.NUM_REFERENCES)]
+                correct_label: str = example["label"]
+                references = [
+                    Reference(Output(text=holdings[i]), tags=([CORRECT_TAG] if correct_label == str(i) else []))
+                    for i in range(self.NUM_REFERENCES)
+                ]
+                instance = Instance(
+                    input=Input(text=citing_prompt),
+                    references=references,
+                    split=helm_split_name,
+                    id=f"id{example_id}",
+                )
+                instances.append(instance)
+        return instances

helm/benchmark/scenarios/chw_care_plan_scenario.py

@@ -0,0 +1,106 @@
+import pandas as pd
+from typing import List
+
+from helm.common.general import check_file_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+
+def create_prompt_text(clinical_note):
+    # Create a prompt for the model to generate a care plan based on the clinical note
+    prompt = f"""
+You are provided with a clinical note regarding a physician-patient interaction. Your task is to \
+extract specific information based solely on the content provided. Do not hallucinate or infer details \
+that are not explicitly stated in the text. Any information you include must be directly entailed by the text.
+
+Instructions:
+
+Extract the required information precisely as presented in the source text.
+
+If the text does not contain specific information, clearly state "Not mentioned".
+
+Maintain the patient's original wording whenever possible.
+
+Response Format:
+
+Chief Complaint
+[ENTER CHIEF COMPLAINT]
+
+History of Present Illness
+
+Onset: When did it start? Did it begin suddenly or gradually? [ENTER ONSET INFORMATION]
+
+Provoking/Palliating Factors: What makes the symptoms better or worse? [ENTER Provoking/Palliating Factors INFORMATION]
+
+Quality: Describe the symptoms, e.g., sharp pain, dull pain, stabbing pain. [ENTER QUALITY INFORMATION]
+
+Region/Radiation: Where are your symptoms located? Do they move? [ENTER REGION/RADIATION INFORMATION]
+
+Severity: On a scale of 1 to 10, how severe are your symptoms? [ENTER SEVERITY INFORMATION]
+
+Timing: When do you experience the symptoms? What times of day? [ENTER TIMING INFORMATION]
+
+Related Symptoms: Are there any other symptoms related to the main complaint? [ENTER RELATED SYMPTOMS INFORMATION]
+
+Ensure your responses are concise, accurate, and entirely supported by the provided text. \
+Do not introduce external knowledge or assumptions.
+
+Clinical Note:
+{clinical_note}
+"""
+    return prompt
+
+
+class CHWCarePlanScenario(Scenario):
+    """
+    A scenario for a dataset containing free form text of a clinical health worker care plan, with the
+    associated goal being to restructure that text into a given format.
+
+    - Input:  The clinical note (column "MO Note").
+    - Output:  The clinical note (column "MO Note"). We will use this note as the reference for entailment.
+    """
+
+    name = "chw_care_plan"
+    description = (
+        "NoteExtract is a benchmark that focuses on the structured extraction of information"
+        "from free-form clinical text. It provides care plan notes authored by health workers"
+        "and evaluates a model's ability to convert them into a predefined structured format,"
+        "such as fields for Chief Complaint and History of Present Illness. The benchmark"
+        "emphasizes faithful extraction without hallucination or inference."
+    )
+    tags = ["question_answering", "biomedical"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[CHWCarePlanScenario] Required data file not found: '{self.data_path}'")
+        df = pd.read_csv(self.data_path)  # columns: ["text", "target", ...]
+
+        instances: List[Instance] = []
+
+        # Use the entire dataset as one split (TEST_SPLIT)
+        for idx, row in df.iterrows():
+            note_text: str = row["MO Note"]
+            prompt_text = create_prompt_text(note_text)
+            if pd.isna(note_text):
+                print(f"Skipping row {idx} due to NaN value in 'MO Note'")
+                continue
+            # print(f"Prompt text: {prompt_text}")
+
+            # Create one Instance per patient
+            instance = Instance(
+                input=Input(text=prompt_text),
+                references=[Reference(Output(text=note_text), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/civil_comments_scenario.py

@@ -4,7 +4,7 @@ from typing import Dict, List, Set
 import pandas as pd
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     TEST_SPLIT,
     TRAIN_SPLIT,

helm/benchmark/scenarios/clear_scenario.py

@@ -0,0 +1,157 @@
+import os
+import pandas as pd
+from typing import List
+
+from helm.common.general import check_file_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+
+class CLEARScenario(Scenario):
+    """
+    CLEARScenario is a dataset of human-labeled medical texts that indicate whether a patient has a history
+    of various medical conditions like alcohol dependence, depression, PTSD etc. Each example includes:
+
+      - text: A medical note or patient report.
+      - result_human: The human-provided label where:
+            1 indicates the patient has a history of the condition,
+            0 indicates the patient does not have a history of the condition,
+            2 indicates uncertainty about the patient's history of the condition.
+
+    For this scenario, the human label is mapped to a multiple-choice option as follows:
+            1 -> A, 0 -> B, 2 -> C
+
+    The task is to classify the text using a multiple-choice format.
+
+    Sample Synthetic Prompt:
+        You are a helpful medical assistant. Determine whether the patient has a history of <medical condition>.
+
+        Text: [insert text here]
+
+        A. Has a history of alcohol dependence
+        B. Does not have a history of alcohol dependence
+        C. Uncertain
+
+        Answer:
+    """
+
+    # List of all available conditions
+    CONDITIONS = [
+        "alcohol_dependence",
+        "attention_deficit_hyperactivity_disorder",
+        "bipolar_disorder",
+        "chronic_pain",
+        "homelessness",
+        "liver_disease",
+        "major_depression",
+        "personality_disorder",
+        "post_traumatic_stress_disorder",
+        "substance_use_disorder",
+        "suicidal_behavior",
+        "tobacco_dependence",
+        "unemployment",
+    ]
+
+    # Map condition names to human-readable descriptions for prompts
+    CONDITION_PROMPTS = {
+        "alcohol_dependence": "alcohol dependence",
+        "attention_deficit_hyperactivity_disorder": "attention deficit hyperactivity disorder (ADHD)",
+        "bipolar_disorder": "bipolar disorder",
+        "chronic_pain": "chronic pain",
+        "homelessness": "homelessness",
+        "liver_disease": "liver disease",
+        "major_depression": "major depression",
+        "personality_disorder": "personality disorder",
+        "post_traumatic_stress_disorder": "post-traumatic stress disorder (PTSD)",
+        "substance_use_disorder": "substance use disorder",
+        "suicidal_behavior": "suicidal behavior",
+        "tobacco_dependence": "tobacco dependence",
+        "unemployment": "unemployment",
+    }
+
+    def __init__(self, condition: str, data_path: str):
+        """Initialize the scenario with a specific medical condition"""
+        super().__init__()
+
+        if condition not in self.CONDITIONS:
+            raise ValueError(f"Condition '{condition}' not supported. Available conditions: {self.CONDITIONS}")
+        self.data_path = data_path
+        self.condition = condition
+        self.name = f"clear_{condition}"
+        self.description = (
+            "CLEAR is a benchmark designed to evaluate models on their ability to detect medical"
+            "conditions from patient notes using categorical responses. Each instance consists of"
+            "a clinical note and a target condition, requiring the model to classify the patient's"
+            "history as either affirmative, negative, or uncertain."
+        )  # noqa: E501
+        self.tags = ["classification", "biomedical", condition.replace("_", "-")]
+
+    def get_answer_choices(self) -> List[str]:
+        """Get the possible answer choices with the condition filled in."""
+        condition_text = self.CONDITION_PROMPTS[self.condition]
+        return [f"Has a history of {condition_text}", f"Does not have a history of {condition_text}", "Uncertain"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Load and process the data for the specified conditon."""
+        excel_path = os.path.join(self.data_path, f"{self.condition}.xlsx")
+        check_file_exists(excel_path, msg=f"[CLEARScenario] Required data file not found: '{excel_path}'")
+
+        df = pd.read_excel(excel_path)
+
+        possible_answer_choices = self.get_answer_choices()
+
+        instances: List[Instance] = []
+        # All instances are assigned to the test split (zero-shot setting)
+        split = TEST_SPLIT
+        condition_text = self.CONDITION_PROMPTS[self.condition]
+
+        for _, row in df.iterrows():
+            text = str(row.get("text", "")).strip()
+            label = str(row.get("result_human", "")).strip()
+
+            # Skip the instance if any required field is missing
+            if not text or not label:
+                continue
+
+            # Map the human label to the multiple-choice option: 1 -> A, 0 -> B, 2 -> C
+            label_mapping = {
+                "1": f"Has a history of {condition_text}",
+                "0": f"Does not have a history of {condition_text}",
+                "2": "Uncertain",
+            }
+            mapped_label = label_mapping.get(label, label)
+
+            # Build the References using the possible answer choices,
+            # marking the correct one with the CORRECT_TAG.
+            references: List[Reference] = [
+                Reference(Output(text=choice), tags=[CORRECT_TAG] if choice == mapped_label else [])
+                for choice in possible_answer_choices
+            ]
+
+            input_text = (
+                f"You are a medical assistant reviewing patient notes. "
+                f"Determine whether the patient has a history of {condition_text}.\n\n"
+                f"Original Clinical Note:\n"
+                f"{text}\n\n"
+                f"A. Has a history of {condition_text}\n"
+                f"B. Does not have a history of {condition_text}\n"
+                f"C. Uncertain\n\n"
+                f"Answer:"
+            )
+
+            instances.append(
+                Instance(
+                    input=Input(text=input_text),
+                    references=references,
+                    split=split,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/cleva_scenario.py

@@ -18,7 +18,7 @@ from helm.common.general import (
     ensure_directory_exists,
 )
 from helm.common.hierarchical_logger import hlog
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,
@@ -29,7 +29,7 @@ from .scenario import (
     Output,
     get_scenario_cache_path,
 )
-from .code_scenario import CodeReference, CodeInstance
+from helm.benchmark.scenarios.code_scenario import CodeReference, CodeInstance
 
 
 CLEVA_DATA_URL = "http://39.108.215.175/data"

helm/benchmark/scenarios/code_scenario.py

@@ -57,9 +57,19 @@ from typing import List, Dict, Iterable, Optional, cast
 
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from .code_scenario_helper import run as run_reindent
-from .code_scenario_apps_pinned_file_order import apps_listdir_with_pinned_order
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.code_scenario_helper import run as run_reindent
+from helm.benchmark.scenarios.code_scenario_apps_pinned_file_order import apps_listdir_with_pinned_order
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 class CodeReference(Reference):
@@ -292,7 +302,10 @@ class CodeScenario(Scenario):
         self.human_eval_hparams = dict(num_train_instances=0, num_val_instances=0, num_test_instances=164)
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        # By construction, output_path == 'benchmark_output/scenarios/code'.
+        # By construction, output_path == args.output_path + '/scenarios/code'
+        # where args.output_path is parsed from the command line argument.
+        # The default self.output_path here is '/benchmark_output/scenarios/ice'.
+        # See helm.benchmark.runner for more details about args.output_path.
         if self.dataset == "humaneval":
             target_path = os.path.join(output_path, "HumanEval.jsonl")
             ensure_file_downloaded(

helm/benchmark/scenarios/commonsense_scenario.py

@@ -4,7 +4,7 @@ from typing import List
 
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.common.hierarchical_logger import hlog
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,

helm/benchmark/scenarios/conv_fin_qa_calc_scenario.py

@@ -0,0 +1,97 @@
+import json
+import os
+from typing import Dict, List, Any
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    CORRECT_TAG,
+    Output,
+)
+from helm.common.general import ensure_file_downloaded
+
+
+class ConvFinQACalcScenario(Scenario):
+    """A mathematical calculation benchmark based on ConvFinQA.
+
+    Data source:
+    https://github.com/czyssrs/ConvFinQA
+
+    Reference:
+    Zhiyu Chen, Shiyang Li, Charese Smiley, Zhiqiang Ma, Sameena Shah, and William Yang Wang. 2022.
+    ConvFinQA: Exploring the Chain of Numerical Reasoning in Conversational Finance Question Answering.
+    In Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing,
+    pages 6279–6292, Abu Dhabi, United Arab Emirates. Association for Computational Linguistics.
+    https://aclanthology.org/2022.emnlp-main.421
+    """  # noqa: E501
+
+    name = "conv_fin_qa_calc"
+    description = "A mathematical calculation benchmark based on ConvFinQA: Exploring the Chain of Numerical Reasoning in Conversational Finance Question Answering [(Chen ey al., 2022)](https://arxiv.org/pdf/2210.03849.pdf)."  # noqa: E501
+    tags = ["question_answering", "finance"]
+
+    DATASET_DOWNLOAD_URL: str = (
+        "https://github.com/czyssrs/ConvFinQA/raw/cf3eed2d5984960bf06bb8145bcea5e80b0222a6/data.zip"
+    )
+
+    _SPLIT_TO_JSON_FILE_NAME: Dict[str, str] = {TRAIN_SPLIT: "train_turn.json", VALID_SPLIT: "dev_turn.json"}
+
+    def make_pseudo_markdown_table(self, table: List[List[Any]], sep: str = "\n") -> str:
+        markdown_lines: List[str] = []
+
+        for row in table:
+            row_inner_markdown = " | ".join([str(cell) for cell in row])
+            row_markdown = f"| {row_inner_markdown} |"
+            markdown_lines.append(row_markdown)
+
+        return sep.join(markdown_lines)
+
+    def convert_to_instance(self, dic: Dict[str, Any], split: str, sep: str = "\n") -> Instance:
+        linearized_table = self.make_pseudo_markdown_table(dic["table"])
+        input_text = f"Table: {sep}{linearized_table}{sep}{sep}"
+
+        if "gold_ind" in dic["annotation"]:
+            facts = dic["annotation"]["gold_ind"]
+        elif "gold_inds" in dic["annotation"]:
+            facts = dic["annotation"]["gold_inds"]
+        else:
+            facts = {}
+        table_text = ""
+        for fact_type, fact in facts.items():
+            if "text" in fact_type:
+                table_text += fact
+        if table_text:
+            input_text += f"Text: {sep}{table_text}{sep}{sep}"
+
+        for ind, q in enumerate(dic["annotation"]["cur_dial"]):
+            if ind < len(dic["annotation"]["cur_dial"]) - 1:
+                input_text += f"Question: {q}{sep}Answer: {dic['annotation']['exe_ans_list'][ind]}{sep}"
+            else:
+                input_text += f"Question: {q}"
+
+        answer = str(dic["annotation"]["exe_ans"])
+        return Instance(
+            input=Input(text=input_text),
+            references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+            split=split,
+        )
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = os.path.join(output_path, "data")
+        ensure_file_downloaded(
+            source_url=self.DATASET_DOWNLOAD_URL,
+            target_path=os.path.join(output_path, "data"),
+            unpack=True,
+            unpack_type="unzip",
+        )
+        instances: List[Instance] = []
+        for split, json_file_name in self._SPLIT_TO_JSON_FILE_NAME.items():
+            json_file_path = os.path.join(data_path, json_file_name)
+            with open(json_file_path) as f:
+                raw_instances = json.load(f)
+                for raw_instance in raw_instances:
+                    instances.append(self.convert_to_instance(raw_instance, split))
+        return instances

helm/benchmark/scenarios/copyright_scenario.py

@@ -4,7 +4,7 @@ import tqdm
 from typing import List
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, CORRECT_TAG, TEST_SPLIT, Input, Output
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Reference, CORRECT_TAG, TEST_SPLIT, Input, Output
 
 datatag2hash_text = {
     # The "average" book.

helm/benchmark/scenarios/covid_dialog_scenario.py

@@ -2,7 +2,16 @@ import os
 from typing import List
 
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, ALL_SPLITS, CORRECT_TAG, VALID_SPLIT, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    ALL_SPLITS,
+    CORRECT_TAG,
+    VALID_SPLIT,
+    Input,
+    Output,
+)
 
 
 class COVIDDialogScenario(Scenario):