crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/scenarios/mmlu_pro_scenario.py

@@ -0,0 +1,95 @@
+from typing import Dict, List
+from datasets import Dataset, load_dataset
+
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MMLUProScenario(Scenario):
+    """
+    The MMLU-Pro dataset is an advanced version of the Massive Multitask Language Understanding (MMLU)
+    benchmark, created to push the boundaries of language models' reasoning and comprehension skills.
+    Designed as a more challenging evaluation, it increases the answer options per question from four
+    to ten, significantly reducing the likelihood of correct random guesses. This update makes the
+    dataset better at distinguishing the capabilities of models on complex tasks.
+
+    MMLU-Pro emphasizes reasoning over simple factual recall by integrating diverse, intricate questions
+    across 14 domains, including subjects like biology, economics, law, and psychology. In addition, it
+    addresses limitations in the original MMLU by filtering out trivial questions, making it a more
+    robust benchmark. Performance comparisons suggest that models benefit from reasoning-based
+    approaches (such as Chain of Thought, or CoT) on MMLU-Pro, which contrasts with the original
+    MMLU where CoT didn’t show as much benefit. This makes MMLU-Pro especially suitable for evaluating
+    advanced models that rely on nuanced reasoning and comprehension skills​.
+
+    Dataset: https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro
+    Paper: https://arxiv.org/abs/2406.01574
+    """
+
+    name = "mmlu_pro"
+    description = "Enhanced Massive Multitask Language Understanding with increased options and reasoning"
+    tags = ["knowledge", "multiple_choice", "reasoning"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        self.subject: str = subject.replace("_", " ")
+
+    def process_dataset(self, data: Dataset, split: str) -> List[Instance]:
+        """
+        Process the dataset to create instances.
+
+        :param data: Hugging Face `Dataset` containing the data for a specific split.
+        :param split: The data split (e.g., "train", "test").
+        :return: A list of processed `Instance` objects.
+        """
+        instances: List[Instance] = []
+        hlog(f"Processing data for {split} split")
+        for row in data:
+            id = row["question_id"]
+            question = row["question"]
+            answers = row["options"]
+            correct_choice = row["answer"]
+            answers_dict = dict(zip(["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"], answers))
+            correct_answer = answers_dict[correct_choice]
+
+            def answer_to_reference(answer: str) -> Reference:
+                return Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+
+            instance = Instance(
+                id=f"id{id}",
+                input=Input(text=question),
+                references=list(map(answer_to_reference, answers)),
+                split=split,
+            )
+            instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Load and process the MMLU-Pro dataset to create instances.
+
+        :param output_path: Path to save or output the processed instances.
+        :return: A list of all processed `Instance` objects.
+        """
+        # Load the MMLU-Pro dataset from Hugging Face
+        dataset = load_dataset("TIGER-Lab/MMLU-Pro", revision="3373e0b")
+
+        # Process all the instances
+        instances: List[Instance] = []
+        splits: Dict[str, str] = {
+            "validation": TRAIN_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for hf_split, split in splits.items():
+            data = dataset[hf_split].filter(lambda x: self.subject == "all" or self.subject == x["category"])
+            instances.extend(self.process_dataset(data, split))
+
+        return instances

helm/benchmark/scenarios/mmlu_scenario.py

@@ -4,7 +4,17 @@ from typing import Dict, List
 
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 class MMLUScenario(Scenario):

helm/benchmark/scenarios/msmarco_scenario.py

@@ -6,7 +6,7 @@ from typing import Dict, List, Tuple, Optional, Union
 
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.common.hierarchical_logger import hlog
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,

helm/benchmark/scenarios/mtsamples_procedures_scenario.py

@@ -0,0 +1,144 @@
+import os
+import requests
+from typing import List
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Output,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class MTSamplesProceduresScenario(Scenario):
+    """
+    Processes the MTSamples Procedure dataset, a subset of MTSamples,
+    specifically focusing on procedure-related medical notes.
+    This dataset contains transcribed medical reports detailing various procedures,
+    treatments, and surgical interventions.
+
+    - Extracts `PLAN`, `SUMMARY`, or `FINDINGS` sections as references.
+    - Ensures these sections are excluded from the input text.
+    - Filters out files that do not contain any of the three reference sections.
+
+    Data source: https://github.com/raulista1997/benchmarkdata/tree/main/mtsample_procedure
+    """
+
+    GIT_HASH = "c4c252443fa9c52afb6960f53e51be278639bea2"
+    GITHUB_DIR_URL = f"https://github.com/raulista1997/benchmarkdata/tree/{GIT_HASH}/mtsample_procedure"
+    RAW_BASE_URL = f"https://raw.githubusercontent.com/raulista1997/benchmarkdata/{GIT_HASH}/mtsample_procedure/"
+
+    name = "mtsamples_procedures"
+    description = (
+        "MTSamples Procedures is a benchmark composed of transcribed operative notes,"
+        "focused on documenting surgical procedures. Each example presents a brief patient case"
+        "involving a surgical intervention, and the model is tasked with generating a coherent"
+        "and clinically accurate procedural summary or treatment plan."
+    )
+    tags = ["medical", "transcription", "plan_generation"]
+
+    def fetch_file_list(self) -> List[str]:
+        """
+        Uses the GitHub API to fetch the list of `.txt` files in the dataset directory.
+        """
+        api_url = "https://api.github.com/repos/raulista1997/benchmarkdata/contents/mtsample_procedure"
+        headers = {"Accept": "application/vnd.github+json"}
+
+        response = requests.get(api_url, headers=headers)
+        if response.status_code != 200:
+            raise Exception(f"Failed to fetch file list from GitHub API ({api_url})")
+
+        files = response.json()
+        return [file["name"] for file in files if file["name"].endswith(".txt")]
+
+    def download_file(self, file_name: str, output_dir: str) -> str:
+        """
+        Downloads a text file from GitHub and saves it locally.
+        """
+        file_url = self.RAW_BASE_URL + file_name
+        file_path = os.path.join(output_dir, file_name)
+
+        if not os.path.exists(file_path):  # Avoid redundant downloads
+            response = requests.get(file_url)
+            if response.status_code != 200:
+                raise Exception(f"Failed to download {file_url}")
+
+            with open(file_path, "w", encoding="utf-8") as f:
+                f.write(response.text)
+
+        return file_path
+
+    def extract_sections(self, text: str) -> tuple:
+        """
+        Extracts `PLAN`, `SUMMARY`, and `FINDINGS` sections from the text.
+        Returns (plan, summary, findings) as a tuple, ensuring uppercase detection.
+        """
+        plan, summary, findings = None, None, None
+        text_upper = text.upper()
+
+        if "PLAN:" in text_upper:
+            plan = text.split("PLAN:")[1].split("\n", 1)[0].strip()
+
+        if "SUMMARY:" in text_upper:
+            summary = text.split("SUMMARY:")[1].split("\n", 1)[0].strip()
+
+        if "FINDINGS:" in text_upper:
+            findings = text.split("FINDINGS:")[1].split("\n", 1)[0].strip()
+
+        return plan, summary, findings
+
+    def remove_sections(self, text: str) -> str:
+        """
+        Removes `PLAN`, `SUMMARY`, and `FINDINGS` sections from the input text.
+        """
+        sections = ["PLAN:", "SUMMARY:", "FINDINGS:"]
+        for section in sections:
+            if section in text:
+                text = text.split(section)[0].strip()  # Keep content before the section
+        return text
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Downloads, processes, and converts MTSamples data into HELM format.
+        """
+        ensure_directory_exists(output_path)
+
+        # Fetch list of available files from GitHub
+        file_list = self.fetch_file_list()
+
+        instances = []
+        for file_name in file_list:
+            try:
+                # Download the text file
+                file_path = self.download_file(file_name, output_path)
+
+                # Read content
+                with open(file_path, "r", encoding="utf-8") as f:
+                    text_content = f.read().strip()
+
+                # Extract structured sections
+                plan, summary, findings = self.extract_sections(text_content)
+
+                # Use plan > summary > findings as reference text
+                reference_text = plan or summary or findings
+                if not reference_text:
+                    continue  # Ignore notes with no reference section
+
+                # Remove structured sections from input
+                cleaned_text = self.remove_sections(text_content)
+
+                # Create HELM instance
+                instances.append(
+                    Instance(
+                        input=Input(text=cleaned_text),  # Processed text without sections
+                        references=[Reference(Output(text=reference_text), tags=[CORRECT_TAG])],
+                        split=TEST_SPLIT,
+                    )
+                )
+            except Exception as e:
+                print(f"Error processing {file_name}: {e}")
+
+        return instances

helm/benchmark/scenarios/mtsamples_replicate_scenario.py

@@ -0,0 +1,142 @@
+import os
+import requests
+from typing import List
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Output,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class MTSamplesReplicateScenario(Scenario):
+    """
+    MTSamples.com is designed to give you access to a big collection of transcribed medical reports.
+    These samples can be used by learning, as well as working medical transcriptionists for their daily
+    transcription needs. We present the model with patient information and request it to generate a corresponding
+    treatment plan.
+
+    Sample Synthetic Prompt:
+    Given various information about a patient, return a reasonable treatment plan for the patient.
+
+    - Extracts `PLAN`, `SUMMARY`, or `FINDINGS` as the reference (PLAN preferred).
+    - Removes `PLAN` from the input text but keeps other sections.
+    - Ignores files that do not contain any of these reference sections.
+    """
+
+    GIT_HASH = "ebc104a4f96c5b7602242f301e081e9934a23344"
+    API_BASE_URL = (
+        f"https://api.github.com/repos/raulista1997/benchmarkdata/contents/mtsamples_processed?ref={GIT_HASH}"
+    )
+    RAW_BASE_URL = f"https://raw.githubusercontent.com/raulista1997/benchmarkdata/{GIT_HASH}/mtsamples_processed/"
+
+    name = "mtsamples_replicate"
+    description = (
+        "MTSamples Replicate is a benchmark that provides transcribed medical reports"
+        "from various specialties. It is used to evaluate a model's ability to generate clinically"
+        "appropriate treatment plans based on unstructured patient documentation"
+    )
+    tags = ["medical", "transcription", "plan_generation"]
+
+    def fetch_file_list(self) -> List[str]:
+        """
+        Uses the GitHub API to fetch the list of `.txt` files at a specific commit.
+        """
+        response = requests.get(self.API_BASE_URL)
+        if response.status_code != 200:
+            raise Exception(f"Failed to fetch file list from GitHub API: {response.text}")
+
+        files = response.json()
+        return [f["name"] for f in files if f["name"].endswith(".txt")]
+
+    def download_file(self, file_name: str, output_dir: str) -> str:
+        """
+        Downloads a text file from GitHub and saves it locally.
+        """
+        file_url = self.RAW_BASE_URL + file_name
+        file_path = os.path.join(output_dir, file_name)
+
+        if not os.path.exists(file_path):
+            response = requests.get(file_url)
+            if response.status_code != 200:
+                raise Exception(f"Failed to download {file_url}")
+            with open(file_path, "w", encoding="utf-8") as f:
+                f.write(response.text)
+
+        return file_path
+
+    def extract_sections(self, text: str) -> tuple:
+        """
+        Extracts `PLAN`, `SUMMARY`, and `FINDINGS` sections from the text.
+        Returns (plan, summary, findings) as a tuple, ensuring uppercase detection.
+        """
+        plan, summary, findings = None, None, None
+        text_upper = text.upper()
+
+        if "PLAN:" in text_upper:
+            plan = text.split("PLAN:")[1].split("\n", 1)[0].strip()
+
+        if "SUMMARY:" in text_upper:
+            summary = text.split("SUMMARY:")[1].split("\n", 1)[0].strip()
+
+        if "FINDINGS:" in text_upper:
+            findings = text.split("FINDINGS:")[1].split("\n", 1)[0].strip()
+
+        return plan, summary, findings
+
+    def remove_plan_section(self, text: str) -> str:
+        """
+        Removes `PLAN:` section from the input text while keeping the rest.
+        """
+        sections = ["PLAN:"]
+        for section in sections:
+            if section in text:
+                text = text.split(section)[0].strip()  # Keep content before PLAN
+        return text
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Downloads, processes, and converts MTSamples data into HELM format.
+        """
+        ensure_directory_exists(output_path)
+
+        # Fetch list of available files from GitHub
+        file_list = self.fetch_file_list()
+
+        instances = []
+        for file_name in file_list:
+            try:
+                # Download the text file
+                file_path = self.download_file(file_name, output_path)
+
+                # Read content
+                with open(file_path, "r", encoding="utf-8") as f:
+                    text_content = f.read().strip()
+
+                # Extract structured sections
+                plan, summary, findings = self.extract_sections(text_content)
+
+                # Use plan > summary > findings as reference text
+                reference_text = plan or summary or findings
+                if not reference_text:
+                    continue  # Ignore notes with no reference section
+
+                # Remove PLAN section from input
+                cleaned_text = self.remove_plan_section(text_content)
+
+                # Create HELM instance
+                instances.append(
+                    Instance(
+                        input=Input(text=cleaned_text),  # Processed text without PLAN
+                        references=[Reference(Output(text=reference_text), tags=[CORRECT_TAG])],
+                        split=TEST_SPLIT,
+                    )
+                )
+            except Exception as e:
+                print(f"Error processing {file_name}: {e}")
+
+        return instances