crfm-helm 0.5.4__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/scenarios/enem_challenge_scenario.py

@@ -0,0 +1,58 @@
+from typing import List, Any
+from pathlib import Path
+from datasets import load_dataset
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Output,
+)
+
+
+class ENEMChallengeScenario(Scenario):
+    """
+    The Exame Nacional do Ensino Médio (ENEM) is an advanced High-School level exam widely applied
+    every year by the Brazilian government to students that wish to undertake a University degree.
+
+    The questions are about all types of intelectual fields and they are divided into four groups
+    that are named as: Humanities, Languages, Sciences and Mathematics.
+
+    This scenario is based on the exams that were applied throughout the years of 2009 and 2023.
+
+    The dataset can be found in this link: https://huggingface.co/datasets/eduagarcia/enem_challenge
+    """
+
+    name = "enem_challenge"
+    description = "ENEM Challenge dataset"
+    tags = ["knowledge", "multiple_choice", "pt-br"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the raw data and read all the dialogues
+        dataset: Any
+        # Read all the instances
+        instances: List[Instance] = []
+        cache_dir = str(Path(output_path) / "data")
+
+        dataset = load_dataset("eduagarcia/enem_challenge", cache_dir=cache_dir)
+        for example in dataset["train"]:
+            question = example["question"]
+            choices = example["choices"]
+            answer = example["answerKey"]
+            # Skipping every canceled question!
+            if answer == "ANULADO":
+                continue
+            answers_dict = dict(zip(choices["label"], choices["text"]))
+            correct_answer = answers_dict[answer]
+
+            def answer_to_reference(answer: str) -> Reference:
+                return Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+
+            instance = Instance(
+                input=Input(text=question), split=TEST_SPLIT, references=list(map(answer_to_reference, choices["text"]))
+            )
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/entity_data_imputation_scenario.py

@@ -5,7 +5,17 @@ from typing import List, Tuple
 
 from helm.common.hierarchical_logger import hlog
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 class EntityDataImputationScenario(Scenario):

helm/benchmark/scenarios/entity_matching_scenario.py

@@ -4,8 +4,18 @@ from typing import Dict, List, Tuple
 
 from helm.common.hierarchical_logger import hlog
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
-from .entity_matching_scenario_fixed_random_state import set_fixed_random_state_for_dataset
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.benchmark.scenarios.entity_matching_scenario_fixed_random_state import set_fixed_random_state_for_dataset
 
 
 class EntityMatchingScenario(Scenario):

helm/benchmark/scenarios/financial_phrasebank_scenario.py

@@ -0,0 +1,94 @@
+import os
+import random
+from typing import List
+
+from helm.common.general import ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    Input,
+    Output,
+)
+
+
+class FinancialPhrasebankScenario(Scenario):
+    """
+    A sentiment classification benchmark based on the dataset from Good Debt or Bad Debt - Detecting Semantic Orientations in Economic Texts [(Malo et al., 2013)](https://arxiv.org/abs/1307.5336).
+
+    Context:
+    Polar sentiment dataset of sentences from financial news. The dataset consists of 4840 sentences from English
+    language financial news categorized by sentiment. The dataset is divided by agreement rate of 5-8 annotators.
+
+    This release of the financial phrase bank covers a collection of 4840 sentences. The selected collection of
+    phrases was annotated by 16 people with adequate background knowledge on financial markets.
+
+    Given the large number of overlapping annotations (5 to 8 annotations per sentence), there are several ways
+    to define a majority vote based gold standard. To provide an objective comparison, the paper authors have formed 4 alternative
+    reference datasets based on the strength of majority agreement: 100%, 75%, 66% and 50%.
+
+    Data source:
+    https://huggingface.co/datasets/takala/financial_phrasebank
+
+    Reference:
+    P. Malo, A. Sinha, P. Korhonen, J. Wallenius, and P. Takala, “Good debt or bad debt: Detecting semantic orientations in economic texts,” Journal of the Association for Information Science and Technology, vol. 65, 2014.
+    https://arxiv.org/pdf/1307.5336
+
+    """  # noqa: E501
+
+    name = "financial_phrasebank"
+    description = "The dataset consists of 4840 sentences from English \
+                   language financial news categorized by sentiment."
+    tags = ["finance", "sentiment analysis", "classification"]
+
+    INSTRUCTIONS = """The dataset consists of sentences from English language financial news categorized by sentiment.
+Classify the sentences into one of the 3 sentiment categories.
+Possible labels:\n1. positive\n2. neutral\n3. negative"""  # noqa: E501
+    DATASET_URL = "https://huggingface.co/datasets/takala/financial_phrasebank/resolve/598b6aad98f7c8d67be161b12a4b5f2497e07edd/data/FinancialPhraseBank-v1.0.zip"  # noqa: E501
+    AGREEMENT_VALUES = [50, 66, 75, 100]
+    TRAIN_SPLIT_SIZE = 0.7
+
+    def __init__(self, agreement: int, random_seed: int = 121):
+        """The initialization of an instance.
+
+        Args:
+            subset: str: This argument is used to specify the ratio of annotators who agreed on the ground truth label.
+            The value must be one of the strings defined in
+            SUBSETS = ["sentences_allagree", "sentences_75agree", "sentences_66agree", "sentences_50agree"].
+            random_seed: int = 121: The random seed for sampling the train/test splits.
+        """
+        super().__init__()
+        if agreement not in self.AGREEMENT_VALUES:
+            raise Exception(
+                f"Unknown `agreement` value: {agreement}, allowed values are {self.AGREEMENT_VALUES}".format(agreement)
+            )
+        self.agreement = agreement
+        self.random_seed = random_seed
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_parent_path = os.path.join(output_path, "data")
+        ensure_file_downloaded(
+            self.DATASET_URL,
+            data_parent_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+        file_name = "Sentences_AllAgree.txt" if self.agreement == 100 else f"Sentences_{self.agreement}Agree.txt"
+        data_file_path = os.path.join(data_parent_path, "FinancialPhraseBank-v1.0", file_name)
+        with open(data_file_path, mode="r", encoding="iso-8859-1") as f:
+            lines = list(f.readlines())
+            random.Random(self.random_seed).shuffle(lines)
+        train_split_index = int(len(lines) * self.TRAIN_SPLIT_SIZE)
+        instances: List[Instance] = []
+        for index, line in enumerate(lines):
+            sentence, label = line.strip().rsplit("@", 1)
+            instance = Instance(
+                input=Input(text=sentence),
+                references=[Reference(Output(text=label), tags=[CORRECT_TAG])],
+                split=TRAIN_SPLIT if index < train_split_index else TEST_SPLIT,
+            )
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/gold_commodity_news_scenario.py

@@ -0,0 +1,124 @@
+import dataclasses
+import math
+import os
+import random
+from typing import List
+
+import pandas as pd
+
+from helm.benchmark.runner import TRAIN_SPLIT
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Instance,
+    Reference,
+    Scenario,
+    Output,
+)
+
+
+class GoldCommodityNewsScenario(Scenario):
+    """Gold commodity news headline classification
+
+    This dataset contains gold commodity news headlines annotated by humans labeled by humans with regards to
+    whether the news headline discusses past movements and expected directionality in prices, asset comparison
+    and other general information. The task is to classify the news headlines using these labels.
+
+    Paper: https://arxiv.org/abs/2009.04202
+    Dataset: https://www.kaggle.com/datasets/daittan/gold-commodity-news-and-dimensions
+
+    Citation:
+    Ankur Sinha, Tanmay Khandait
+    "Impact of News on the Commodity Market: Dataset and Results." arXiv preprint arXiv:2009.04202 (2020)"""
+
+    name = "gold_commodity_news"
+    description = "The dataset is a collection of news items related to the gold commodities from various sources."
+
+    tags = ["news headline", "classification"]
+
+    CATEGORY_COLUMN_NAMES = {
+        "price_or_not": "Price or Not",
+        "direction_up": "Direction Up",
+        "direction_constant": "Direction Constant",
+        "direction_down": "Direction Down",
+        "past_price": "PastPrice",
+        "future_price": "FuturePrice",
+        "past_news": "PastNews",
+        "future_news": "FutureNews",
+        "assert_comparison": "Asset Comparision",
+    }
+
+    CATEGORY_INSTRUCTIONS = {
+        "price_or_not": "the gold price",
+        "direction_up": "the gold price heading up",
+        "direction_constant": "the price remaining constant or stable",
+        "direction_down": "the gold price heading down",
+        "past_price": "any past information about gold prices",
+        "future_price": "any future information about gold prices",
+        "past_news": "any past information other than the gold prices",
+        "future_news": "any future information other than the gold prices",
+        "assert_comparison": "a comparison purely in the context of the gold commodity with another asset",
+    }
+
+    @classmethod
+    def get_instructions(cls, category: str):
+        if category not in GoldCommodityNewsScenario.CATEGORY_INSTRUCTIONS:
+            raise ValueError(
+                f"Invalid category: '{category}' "
+                f"Valid categories are: {list(GoldCommodityNewsScenario.CATEGORY_INSTRUCTIONS.keys())}"
+            )
+
+        return (
+            "The following are news headlines about the gold commodity. "
+            "Classify whether the news headline discusses "
+            f'{GoldCommodityNewsScenario.CATEGORY_INSTRUCTIONS[category]}. Answer only "Yes" or "No".'
+        )
+
+    def __init__(self, category: str):
+        super().__init__()
+        if category not in GoldCommodityNewsScenario.CATEGORY_INSTRUCTIONS:
+            raise ValueError(
+                f"Invalid category: '{category}' "
+                f"Valid categories are: {list(GoldCommodityNewsScenario.CATEGORY_INSTRUCTIONS.keys())}"
+            )
+        self.column_name = GoldCommodityNewsScenario.CATEGORY_COLUMN_NAMES[category]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        ensure_directory_exists(output_path)
+        data_path = os.path.join(output_path, "finalDataset_0208.csv")
+
+        ensure_file_downloaded(
+            source_url=(
+                "https://www.kaggle.com/api/v1/datasets/download/daittan/"
+                "gold-commodity-news-and-dimensions?dataset_version_number=1"
+            ),
+            target_path=data_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+
+        df = pd.read_csv(data_path)
+
+        instances: List[Instance] = []
+        for _, row in df.iterrows():
+            expected_output: str
+            if row[self.column_name] == 1:
+                expected_output = "Yes"
+            else:
+                expected_output = "No"
+
+            instance = Instance(
+                input=Input(text=str(row["News"])),
+                references=[Reference(Output(text=expected_output), tags=[CORRECT_TAG])],
+                split=str(TEST_SPLIT),
+            )
+            instances.append(instance)
+
+        # no explicit train/test split, so randomly assign 10% of rows to train
+        random.seed(0)
+        train_indexes = random.sample(list(range(len(instances))), k=math.floor(len(instances) / 10))
+        for train_index in train_indexes:
+            instances[train_index] = dataclasses.replace(instances[train_index], split=TRAIN_SPLIT)
+        return instances

helm/benchmark/scenarios/gpqa_scenario.py

@@ -0,0 +1,80 @@
+import datasets
+import os
+import random
+from typing import List
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.general import ensure_directory_exists
+
+
+SUBSETS = ["gpqa_main", "gpqa_diamond", "gpqa_extended"]
+
+# Train example indices below are found by indexing examples given in the original paper repo
+TRAIN_EXAMPLE_INDICES = {
+    "gpqa_main": [339, 105],
+    "gpqa_diamond": [124, 39],
+    "gpqa_extended": [146, 330, 436],
+}
+
+
+class GPQAScenario(Scenario):
+    """GPQA
+
+    GPQA is a multiple-choice, Q&A dataset of very hard questions written and validated by experts in biology, physics,
+    and chemistry. When attempting questions out of their own domain (e.g., a physicist answers a chemistry question),
+    these experts get only 34% accuracy, despite spending >30m with full access to Google."""
+
+    name = "gpqa"
+    description = "A Graduate-Level Google-Proof Q&A Benchmark"
+    tags = ["question answering"]
+
+    def __init__(self, subset: str, random_seed=42):
+        super().__init__()
+        assert subset in SUBSETS, "Unknown subset: {}".format(subset)
+        self.subset = subset
+        self.random_seed = random_seed
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Get GPQA from HuggingFace
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "Idavidrein/gpqa",
+            self.subset,
+            trust_remote_code=True,
+            cache_dir=cache_dir,
+            split="train",
+            revision="90b8e5be2b1d3d2dbfe016cdab47981150600c4a",
+        )
+        assert isinstance(dataset, datasets.Dataset)
+
+        # Read all instances
+        random.seed(self.random_seed)
+        instances: List[Instance] = []
+        for idx, row in enumerate(dataset):
+            input = Input(text=row["Question"].strip())
+            references = [
+                Reference(Output(text=row["Correct Answer"].strip()), tags=[CORRECT_TAG]),
+                Reference(Output(text=row["Incorrect Answer 1"].strip()), tags=[]),
+                Reference(Output(text=row["Incorrect Answer 2"].strip()), tags=[]),
+                Reference(Output(text=row["Incorrect Answer 3"].strip()), tags=[]),
+            ]
+            random.shuffle(references)
+            if idx in TRAIN_EXAMPLE_INDICES[self.subset]:
+                extra_data = {
+                    "chain_of_thought": row["Explanation"],
+                }
+                instance = Instance(input=input, references=references, split=TRAIN_SPLIT, extra_data=extra_data)
+            else:
+                instance = Instance(input=input, references=references, split=TEST_SPLIT)
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/grammar_scenario.py

@@ -1,7 +1,7 @@
 from typing import List
 
-from .scenario import Scenario, Instance, Input, TEST_SPLIT
-from .grammar import read_grammar, generate_derivations, Derivation, get_values, get_tags
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+from helm.benchmark.scenarios.grammar import read_grammar, generate_derivations, Derivation, get_values, get_tags
 
 
 class GrammarScenario(Scenario):

helm/benchmark/scenarios/gsm_scenario.py

@@ -3,7 +3,16 @@ import os
 from typing import Dict, List
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, CORRECT_TAG, TRAIN_SPLIT, TEST_SPLIT, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    Input,
+    Output,
+)
 
 
 class GSM8KScenario(Scenario):

helm/benchmark/scenarios/harm_bench_gcg_transfer_scenario.py

@@ -0,0 +1,50 @@
+import os
+import pandas as pd
+from typing import List
+
+from helm.common.general import ensure_file_downloaded
+
+from .scenario import Scenario, Instance, Input, TEST_SPLIT, Reference, Output
+
+
+class HarmBenchGCGTransferScenario(Scenario):
+    """
+    HarmBenchGCG-T is a standardized evaluation framework for automated red teaming.
+    HarmBench identifies key considerations previously unaccounted for in red teaming
+    evaluations and systematically designed prompts that meet these criteria.
+
+    https://arxiv.org/abs/2402.04249
+    """
+
+    name = "harm_bench_gcg_transfer"
+    description = "HarmBench GCG-T dataset - standard and contextual attacks"
+    tags = ["safety"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = os.path.join(output_path, "data")
+        os.makedirs(data_path, exist_ok=True)
+        url = "https://raw.githubusercontent.com/farzaank/harmbench-gcg-ensembled/refs/heads/main/output.csv"  # noqa: E501
+        outf_path = os.path.join(data_path, "harmbench_gcg.csv")
+        ensure_file_downloaded(
+            source_url=url,
+            target_path=outf_path,
+            unpack=False,
+        )
+
+        df = pd.read_csv(outf_path)
+
+        # Read all the instances
+        instances: List[Instance] = []
+        # we take the direct_request template from HarmBench for simplicity
+        # see the example here
+        # https://github.com/centerforaisafety/HarmBench/blob/main/baselines/direct_request/direct_request.py
+        #
+        # in the future we can use other types of templates including various red team attacks
+        for i, row in df.iterrows():
+            references = [Reference(output=Output(text="unsafe behavior"), tags=[])]
+            input_text = row["behavior_prompt"]
+            input = Input(text=input_text)
+            id = row["behavior_name"]
+            instance = Instance(input=input, split=TEST_SPLIT, references=references, id=id)
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/harm_bench_scenario.py

@@ -4,7 +4,7 @@ from typing import List
 
 from helm.common.general import ensure_file_downloaded
 
-from .scenario import Scenario, Instance, Input, TEST_SPLIT, Reference, Output
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, Reference, Output
 
 
 class HarmBenchScenario(Scenario):

helm/benchmark/scenarios/headqa_scenario.py

@@ -0,0 +1,131 @@
+import os
+from typing import List, Optional
+
+from datasets import DatasetDict, load_dataset
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Instance,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class HeadQAScenario(Scenario):
+    """
+    From "HEAD-QA: A Healthcare Dataset for Complex Reasoning" (Vilares et al.), HEAD-QA is a multi-choice
+    question-answering dataset designed to evaluate reasoning on challenging healthcare-related questions.
+    The questions are sourced from Spanish healthcare exams for specialized positions, covering various topics
+    such as Medicine, Nursing, Psychology, Chemistry, Pharmacology, and Biology.
+
+    Example from the dataset:
+
+    Question:
+    The excitatory postsynaptic potentials:
+
+    A) They are all or nothing.
+    B) They are hyperpolarizing.
+    C) They can be added.
+    D) They spread long distances.
+
+    Answer:
+    The answer is C. Explanation: None provided in this dataset.
+
+    @InProceedings{HEAD-QA,
+    author = {David Vilares and Manuel Vilares and Carlos Gómez-Rodríguez},
+    title = {HEAD-QA: A Healthcare Dataset for Complex Reasoning},
+    year = {2019},
+    abstract = {We present HEAD-QA, a multi-choice question answering testbed to encourage research on complex
+    reasoning. The questions come from exams to access a specialized position in the Spanish healthcare system,
+    and are challenging even for highly specialized humans. We then consider monolingual (Spanish) and
+    cross-lingual (to English) experiments with information retrieval and neural techniques. We show that:
+    (i) HEAD-QA challenges current methods, and (ii) the results lag well behind human performance,
+    demonstrating its usefulness as a benchmark for future work.}}
+
+
+    Task:
+    Given a question and its multiple-choice answers, models must identify the correct answer, corresponding to the
+    `ra` field in the dataset. The dataset spans six healthcare domains and is challenging even for experts.
+    """
+
+    HUGGING_FACE_DATASET_PATH: str = "dvilares/head_qa"
+    SKIP_VQA: bool = True
+    SKIP_TEXTQA: bool = False
+
+    name = "head_qa"
+    description = "A collection of biomedical multiple-choice questions for testing medical knowledge."
+    tags = ["question_answering", "biomedical", "medicine"]
+
+    def __init__(self, language: str = "en", category: Optional[str] = None):
+        """Initialize the HEAD-QA scenario.
+
+        Args:
+            language (str, optional): Language of the dataset. Defaults to "en".
+            category (str, optional): Category of the dataset. If None, all categories are used.
+        """
+        super().__init__()
+        self.language: str = language
+        self.category: Optional[str] = category
+        assert (
+            self.SKIP_VQA or self.SKIP_TEXTQA
+        ), "Failed to initialize HeadQAScenario, one of `SKIP_VQA` or `SKIP_TEXTQA` must be True."
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path: str = os.path.join(output_path, "data")
+        ensure_directory_exists(data_path)
+        dataset: DatasetDict = load_dataset(self.HUGGING_FACE_DATASET_PATH, self.language)
+
+        # XXX: Should we consider validation as test too?
+        # splits = {TRAIN_SPLIT: ["train", "validation"], TEST_SPLIT: ["test"]}
+        # Limit to zero shot setting
+        splits = {TEST_SPLIT: ["test"]}
+        instances: List[Instance] = []
+        for (
+            helm_split_name,
+            dataset_splits_name,
+        ) in splits.items():  # Iterate over the splits
+            for dataset_split_name in dataset_splits_name:
+                split_data = dataset[dataset_split_name]
+
+                for example in split_data:
+                    # Whether to process Visual Question Answering (VQA) examples
+                    if self.SKIP_VQA and example["image"] is not None:
+                        continue
+
+                    # Whether to process Text Question Answering (TextQA) examples
+                    if self.SKIP_TEXTQA and example["image"] is None:
+                        continue
+
+                    # If specified, filter by category
+                    if self.category is not None:
+                        if example["category"] != self.category:
+                            continue
+
+                    question = example["qtext"]
+
+                    # Format the final answer with explanation
+                    instances.append(
+                        Instance(
+                            input=Input(text=question),
+                            references=[
+                                Reference(
+                                    Output(text=option["atext"]),
+                                    tags=[CORRECT_TAG] if option["aid"] == example["ra"] else [],
+                                )
+                                for option in example["answers"]
+                            ],
+                            split=helm_split_name,
+                            extra_data={
+                                "id": example["qid"],
+                                "name": example["name"],
+                                "category": example["category"],
+                                "year": example["year"],
+                            },
+                        )
+                    )
+
+        return instances

helm/benchmark/scenarios/helpdesk_call_summarization_scenario.py

@@ -0,0 +1,37 @@
+import csv
+import os
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    Input,
+)
+
+
+_DATA_DIRRECTORY_PATH = "restricted/helpdesk_call_summarization/HELM Sample Transcripts_20241221_0045"
+
+
+class HelpdeskCallSummarizationScenario(Scenario):
+    """Helpdesk call summarization."""
+
+    name = "helpdesk_call_summarization"
+    description = "Helpdesk call summarization."
+    tags = ["helpdesk_call_center"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        for file_name in os.listdir(_DATA_DIRRECTORY_PATH):
+            if not file_name.endswith(".csv") or not file_name.startswith("Call1-"):
+                continue
+            file_path = os.path.join(_DATA_DIRRECTORY_PATH, file_name)
+            with open(file_path) as f:
+                csv_reader = csv.reader(f)
+                prompt_lines = [f"{row[0]}: {row[4]}" for row in csv_reader]
+            prompt = "\n".join(prompt_lines)
+            instance_id = file_name.removeprefix("Call1-").removesuffix(".csv")
+            input = Input(text=prompt)
+            instance = Instance(id=instance_id, input=input, references=[], split=TEST_SPLIT)
+            instances.append(instance)
+        return instances