crfm-helm 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/scenarios/audio_language/voxceleb2_scenario.py

@@ -0,0 +1,106 @@
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from tqdm import tqdm
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+from helm.common.audio_utils import use_ffmpeg_to_convert_audio_file
+import pandas as pd
+
+
+class VoxCeleb2Scenario(Scenario):
+    """VoxCeleb2
+
+    VoxCeleb2 is an audio-visual dataset consisting of short clips of human speech, extracted from
+    interview videos uploaded to YouTube. This dataset contains over a million utterances from over
+    6,000 speakers.
+
+    Paper: https://www.robots.ox.ac.uk/~vgg/publications/2018/Chung18a/chung18a.pdf
+
+    Citation:
+    @inproceedings{Chung18b,
+        author = "Chung, J.~S. and Nagrani, A. and Zisserman, A.",
+        title = "VoxCeleb2: Deep Speaker Recognition",
+        booktitle = "INTERSPEECH",
+        year = "2018",
+    }
+    """
+
+    DOWNLOADING_URL = "https://huggingface.co/datasets/ProgramComputer/voxceleb/resolve/main/vox2/vox2_test_aac.zip"
+    REFERENCE_URL = (
+        "https://huggingface.co/datasets/LAOS-Y/VoxCeleb2-AudioIdentity/resolve/main/voxceleb2_audioidentity.csv"
+    )
+    IDENTITY_INSTRUCTION = (
+        "Listen to the audio and take your best guess to determine if the two speakers are the same person. "
+        "Give just the letter of your answer and nothing else."
+    )
+
+    name = "voxceleb2"
+    description = (
+        "A large-scale dataset of over a million utterances from over 6,000 speakers with their"
+        "gender, race, identity information"
+        "([Chung et al, 2018](https://www.robots.ox.ac.uk/~vgg/publications/2018/Chung18a/chung18a.pdf))."
+    )
+    tags: List[str] = ["audio", "identification"]
+    options: List[str] = ["Yes", "No"]
+
+    def _convert_answer_to_label(self, answer: bool) -> str:
+        if answer:
+            return "A"
+        else:
+            return "B"
+
+    def _reformat_and_convert_audio_file(
+        self, ori_file_path: str, tgt_audio_data_path: str, audio_data_path: str
+    ) -> str:
+        tgt_audio_path = os.path.join(tgt_audio_data_path, ori_file_path.split(".m4a")[0] + ".wav")
+        ensure_directory_exists(os.path.dirname(tgt_audio_path))
+        use_ffmpeg_to_convert_audio_file(os.path.join(audio_data_path, ori_file_path), tgt_audio_path)
+        return tgt_audio_path
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        audio_data_path = os.path.join(output_path, "audio_files")
+        tgt_audio_data_path = os.path.join(output_path, "tgt_audio_files")
+        ensure_file_downloaded(source_url=VoxCeleb2Scenario.DOWNLOADING_URL, target_path=audio_data_path, unpack=True)
+        annotations = pd.read_csv(VoxCeleb2Scenario.REFERENCE_URL, sep=",")
+        instances = []
+        for _, row in tqdm(annotations.iterrows(), total=len(annotations)):
+            tgt_first_audio_path = self._reformat_and_convert_audio_file(
+                row["first"], tgt_audio_data_path, audio_data_path
+            )
+            tgt_second_audio_path = self._reformat_and_convert_audio_file(
+                row["second"], tgt_audio_data_path, audio_data_path
+            )
+
+            answer = self._convert_answer_to_label(row["same"])
+            # The given correct answer is a letter, but we need an index
+            correct_answer_index: int = ord(answer) - ord("A")
+            references: List[Reference] = []
+            for i, option in enumerate(self.options):
+                reference: Reference
+                is_correct: bool = i == correct_answer_index
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if is_correct else [])
+                references.append(reference)
+
+            input = Input(
+                multimedia_content=MultimediaObject(
+                    [
+                        MediaObject(content_type="audio/wav", location=tgt_first_audio_path),
+                        MediaObject(content_type="audio/wav", location=tgt_second_audio_path),
+                        MediaObject(content_type="text/plain", text=self.IDENTITY_INSTRUCTION),
+                    ]
+                )
+            )
+            instances.append(Instance(input=input, references=references, split=TEST_SPLIT))
+
+        return instances

helm/benchmark/scenarios/autobencher_capabilities_scenario.py

@@ -0,0 +1,68 @@
+import datasets
+import os
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    Input,
+    Output,
+)
+from helm.common.general import ensure_directory_exists
+from helm.common.hierarchical_logger import hlog
+
+
+class AutoBencherCapabilitiesScenario(Scenario):
+    """AutoBencher Capabilities
+
+    AutoBencher uses a language model to automatically search
+    for datasets. AutoBencher Capabilities consists of question
+    answering datasets for math, multilingual, and knowledge-intensive
+    question answering created by AutoBencher.
+
+    Paper: https://arxiv.org/abs/2407.08351"""
+
+    name = "autobencher_capabilities"
+    description = (
+        "AutoBencher Capabilities consists of question answering datasets "
+        "for math, multilingual, and knowledge-intensive "
+        "question answering created by AutoBencher. "
+        "([paper](https://arxiv.org/abs/2407.08351))"
+    )
+    tags = ["question answering"]
+
+    SUBJECTS = ["math", "mt", "econ", "science", "history"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        if subject not in self.SUBJECTS:
+            raise ValueError(f"Unexpected subject {subject}, available subjects are {self.SUBJECTS}")
+        self.subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+
+        # TODO: Switch this to the production dataset when available.
+        dataset = datasets.load_dataset(
+            "xlisali1/AutoBencher-capability.json",
+            split="train",  # Use train split as test, so only zero-shot is supported
+            cache_dir=cache_dir,
+            revision="efe58dd72b6423e3f5c967f16cbea8cce3a51933",
+        )
+        instances: List[Instance] = []
+        for row in dataset:
+            if row["subject"] == self.subject:
+                continue
+            input = Input(text=row["question"])
+            # References are category ID, followed by level 2, 3 and 4 category names.
+            references = [Reference(output=Output(text=row["gold_answer"]), tags=[CORRECT_TAG])]
+            if row["gold_answer"] is None:
+                hlog(f"WARNING: Row had no gold_answer: {row}")
+                continue
+            instance = Instance(input=input, references=references, split=TEST_SPLIT)
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/autobencher_safety_scenario.py

@@ -0,0 +1,51 @@
+import json
+import os
+import pandas as pd
+from typing import List
+
+from helm.common.general import ensure_file_downloaded
+
+from .scenario import CORRECT_TAG, Scenario, Instance, Input, TEST_SPLIT, Reference, Output
+
+
+class AutobencherSafetyScenario(Scenario):
+    """
+    Autobencher safety scenario
+
+    AutoBencher uses a language model to automatically search
+    for datasets. AutoBencher Capabilities consists of question
+    answering datasets for math, multilingual, and knowledge-intensive
+    question answering created by AutoBencher.
+
+    Paper: https://arxiv.org/abs/2407.08351
+    """
+
+    name = "autobencher_safety"
+    description = "Autobencher safety consists of question answering datasets"
+    tags = ["safety"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = os.path.join(output_path, "data")
+        os.makedirs(data_path, exist_ok=True)
+        url = "https://raw.githubusercontent.com/farzaank/AutoBencher/refs/heads/main/safety_processing/process%20full%20dataset%20for%20mTurk/full_dataset.json"  # noqa: E501
+        outf_path = os.path.join(data_path, "full_dataset.json")
+        ensure_file_downloaded(
+            source_url=url,
+            target_path=outf_path,
+            unpack=False,
+        )
+
+        json_data = json.loads(outf_path)
+        df = pd.DataFrame(json_data)
+
+        # Read all the instances
+        instances: List[Instance] = []
+
+        for i, row in df.iterrows():
+            references = [Reference(output=Output(text=row["gold_answer"]), tags=[CORRECT_TAG])]
+            input_text = row["question"]
+            input = Input(text=input_text)
+            id = str(row["category"]) + str(row["id"])
+            instance = Instance(input=input, split=TEST_SPLIT, references=references, id=id)
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/babi_qa_scenario.py

@@ -2,7 +2,7 @@ import os
 from typing import List
 
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,

helm/benchmark/scenarios/banking77_scenario.py

@@ -37,7 +37,12 @@ class Banking77Scenario(Scenario):
         ensure_directory_exists(cache_dir)
 
         # TODO: Switch this to the production dataset when available.
-        dataset = datasets.load_dataset("PolyAI/banking77", cache_dir=cache_dir)
+        dataset = datasets.load_dataset(
+            "PolyAI/banking77",
+            cache_dir=cache_dir,
+            revision="90d4e2ee5521c04fc1488f065b8b083658768c57",
+            trust_remote_code=True,
+        )
 
         instances: List[Instance] = []
         for split_name in [TRAIN_SPLIT, TEST_SPLIT]:

helm/benchmark/scenarios/bbq_scenario.py

@@ -4,7 +4,7 @@ import random
 from typing import List, Dict, Tuple
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,

helm/benchmark/scenarios/big_bench_scenario.py

@@ -5,7 +5,17 @@ from typing import List, Dict
 from urllib.parse import urljoin
 
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, Input, CORRECT_TAG, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    Input,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    Output,
+)
 
 
 class BIGBenchScenario(Scenario):

helm/benchmark/scenarios/bigcodebench_scenario.py

@@ -0,0 +1,58 @@
+import datasets
+import os
+from typing import List
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    Input,
+)
+from helm.common.general import ensure_directory_exists
+
+
+VERSIONS = ["v0.1.0_hf", "v0.1.1", "v0.1.2", "v0.1.3"]
+
+
+class BigCodeBenchScenario(Scenario):
+    """BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions
+
+    BigCodeBench is an easy-to-use benchmark for solving practical and challenging tasks via code.
+    It aims to evaluate the true programming capabilities of large language models (LLMs) in a more realistic setting.
+    The benchmark is designed for HumanEval-like function-level code generation tasks,
+    but with much more complex instructions and diverse function calls."""
+
+    name = "bigcodebench"
+    description = "Benchmarking Code Generation with Diverse Function Calls and Complex Instructions"
+    tags = ["coding"]
+
+    def __init__(self, version: str):
+        super().__init__()
+        assert version in VERSIONS, "Unknown version: {}".format(version)
+        self.version = version
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Get BigCodeBench from HuggingFace
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "bigcode/bigcodebench",
+            revision="057dd1a30dd73d4ed59cfbaaae049870491fa4d6",
+            cache_dir=cache_dir,
+            split=self.version,
+        )
+        assert isinstance(dataset, datasets.Dataset)
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row in dataset:
+            id = row["task_id"]
+            input = Input(text=row["instruct_prompt"])
+            instance = Instance(
+                id=id,
+                input=input,
+                references=[],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/bird_sql_scenario.py

@@ -0,0 +1,94 @@
+import json
+import os
+from typing import Dict, List
+
+from filelock import FileLock
+
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded, shell
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.bird_sql_scenario_helper import (  # type: ignore
+    generate_comment_prompt,
+    generate_schema_prompt,
+)
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Scenario,
+    Instance,
+    Reference,
+    VALID_SPLIT,
+    Input,
+    Output,
+)
+
+
+def _ensure_file_unzipped(source_path: str, target_path: str):
+    with FileLock(f"{target_path}.lock"):
+        if os.path.exists(target_path):
+            hlog(f"Not decompressing {source_path} because {target_path} already exists")
+            return
+        tmp_path = target_path + ".tmp"
+        ensure_directory_exists(tmp_path)
+        shell(["unzip", source_path, "-d", tmp_path])
+        shell(["mv", tmp_path, target_path])
+
+
+class BIRDSQLScenario(Scenario):
+    """BIRD-SQL (Dev)"""
+
+    name = "bird_sql"
+    description = "bird_sql"
+    tags = ["sql"]
+
+    COT_PROMPT = """
+Think step by step, then generate a single SQL query in valid SQLite syntax. Respond with only your reasoning and SQL query in the following tag-delimited format:
+
+<reasoning>
+INSERT_YOUR_REASONING_HERE
+</reasoning>
+<sql>
+INSERT_YOUR_SQL_QUERY_HERE
+</sql>"""  # noqa: E501
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_root_path = os.path.join(output_path, "dev")
+        ensure_file_downloaded(
+            "https://bird-bench.oss-cn-beijing.aliyuncs.com/dev.zip", data_root_path, unpack=True, unpack_type="unzip"
+        )
+        databases_unzip_target = os.path.join(data_root_path, "unzipped_dev_databases")
+        _ensure_file_unzipped(os.path.join(data_root_path, "dev_databases.zip"), databases_unzip_target)
+        # Note: Zip file contains .DS_Store file at the root, which makes dev_databases unzip into a nested directory
+        databases_root_path = os.path.join(databases_unzip_target, "dev_databases")
+
+        database_schema_prompts: Dict[str, str] = {}
+        for database_name in os.listdir(databases_root_path):
+            database_path = os.path.join(databases_root_path, database_name, f"{database_name}.sqlite")
+            print(database_path)
+            if not os.path.exists(database_path):
+                # Ignore stray ".DS_Store" directory
+                continue
+
+            database_schema_prompt = generate_schema_prompt(database_path, num_rows=None)
+            database_schema_prompts[database_name] = database_schema_prompt
+
+        instances: List[Instance] = []
+        dataset_path = os.path.join(data_root_path, "dev.json")
+        dataset = json.load(open(dataset_path, "r"))
+        for row in dataset:
+            question_id: int = row["question_id"]
+            database_id: str = row["db_id"]
+            question: str = row["question"]
+            knowledge: str = row["evidence"]
+            gold_sql: str = row["SQL"]
+
+            schema_prompt = database_schema_prompts[database_id]
+            comment_prompt = generate_comment_prompt(question, knowledge)
+            combined_prompt = schema_prompt + "\n\n" + comment_prompt + self.COT_PROMPT
+            instance = Instance(
+                id=f"id{question_id}",
+                input=Input(text=combined_prompt),
+                references=[Reference(output=Output(text=gold_sql), tags=[CORRECT_TAG])],
+                extra_data={"db_id": row["db_id"]},
+                split=VALID_SPLIT,
+            )
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/bird_sql_scenario_helper.py

@@ -0,0 +1,118 @@
+# flake8: noqa
+# type: ignore
+# fmt: off
+
+from typing import Dict
+import os
+import sqlite3
+
+
+# The following code is copied verbatim from:
+# https://github.com/AlibabaResearch/DAMO-ConvAI/blob/90a76ef9ef1e2960c9bdfa38c63cc81b841e813e/bird/llm/src/gpt_request.py
+# under the following license:
+#
+# MIT License
+#
+# Copyright (c) 2022 Alibaba Research
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+
+def get_db_schemas(bench_root: str, db_name: str) -> Dict[str, str]:
+    """
+    Read an sqlite file, and return the CREATE commands for each of the tables in the database.
+    """
+    asdf = 'database' if bench_root == 'spider' else 'databases'
+    with sqlite3.connect(f'file:{bench_root}/{asdf}/{db_name}/{db_name}.sqlite?mode=ro', uri=True) as conn:
+        # conn.text_factory = bytes
+        cursor = conn.cursor()
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+        tables = cursor.fetchall()
+        schemas = {}
+        for table in tables:
+            cursor.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='{}';".format(table[0]))
+            schemas[table[0]] = cursor.fetchone()[0]
+
+        return schemas
+
+def nice_look_table(column_names: list, values: list):
+    rows = []
+    # Determine the maximum width of each column
+    widths = [max(len(str(value[i])) for value in values + [column_names]) for i in range(len(column_names))]
+
+    # Print the column names
+    header = ''.join(f'{column.rjust(width)} ' for column, width in zip(column_names, widths))
+    # print(header)
+    # Print the values
+    for value in values:
+        row = ''.join(f'{str(v).rjust(width)} ' for v, width in zip(value, widths))
+        rows.append(row)
+    rows = "\n".join(rows)
+    final_output = header + '\n' + rows
+    return final_output
+
+def generate_schema_prompt(db_path, num_rows=None):
+    # extract create ddls
+    '''
+    :param root_place:
+    :param db_name:
+    :return:
+    '''
+    full_schema_prompt_list = []
+    conn = sqlite3.connect(db_path)
+    # Create a cursor object
+    cursor = conn.cursor()
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+    tables = cursor.fetchall()
+    schemas = {}
+    for table in tables:
+        if table == 'sqlite_sequence':
+            continue
+        cursor.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='{}';".format(table[0]))
+        create_prompt = cursor.fetchone()[0]
+        schemas[table[0]] = create_prompt
+        if num_rows:
+            cur_table = table[0]
+            if cur_table in ['order', 'by', 'group']:
+                cur_table = "`{}`".format(cur_table)
+
+            cursor.execute("SELECT * FROM {} LIMIT {}".format(cur_table, num_rows))
+            column_names = [description[0] for description in cursor.description]
+            values = cursor.fetchall()
+            rows_prompt = nice_look_table(column_names=column_names, values=values)
+            verbose_prompt = "/* \n {} example rows: \n SELECT * FROM {} LIMIT {}; \n {} \n */".format(num_rows, cur_table, num_rows, rows_prompt)
+            schemas[table[0]] = "{} \n {}".format(create_prompt, verbose_prompt)
+
+    for k, v in schemas.items():
+        full_schema_prompt_list.append(v)
+
+    schema_prompt = "\n\n".join(full_schema_prompt_list)
+
+    return schema_prompt
+
+def generate_comment_prompt(question, knowledge=None):
+    pattern_prompt_no_kg = "-- Using valid SQLite, answer the following questions for the tables provided above."
+    pattern_prompt_kg = "-- Using valid SQLite and understading External Knowledge, answer the following questions for the tables provided above."
+    # question_prompt = "-- {}".format(question) + '\n SELECT '
+    question_prompt = "-- {}".format(question)
+    knowledge_prompt = "-- External Knowledge: {}".format(knowledge)
+
+    if not knowledge_prompt:
+        result_prompt = pattern_prompt_no_kg + '\n' + question_prompt
+    else:
+        result_prompt = knowledge_prompt + '\n' + pattern_prompt_kg + '\n' + question_prompt
+
+    return result_prompt
+
+def cot_wizard():
+    cot = "\nGenerate the SQL after thinking step by step: "
+    
+    return cot

helm/benchmark/scenarios/blimp_scenario.py

@@ -4,7 +4,7 @@ from typing import List
 
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from .scenario import CORRECT_TAG, Reference, Scenario, Instance, Input, TEST_SPLIT, Output
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Reference, Scenario, Instance, Input, TEST_SPLIT, Output
 
 
 class BLiMPScenario(Scenario):

helm/benchmark/scenarios/bold_scenario.py

@@ -4,7 +4,7 @@ import random
 from typing import List, Dict, Tuple
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     TRAIN_SPLIT,

helm/benchmark/scenarios/boolq_scenario.py

@@ -3,7 +3,7 @@ import os
 from typing import List, Dict, Optional
 
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,

helm/benchmark/scenarios/casehold_scenario.py

@@ -0,0 +1,79 @@
+from typing import List
+import os
+import os.path
+
+from datasets import load_dataset, DatasetDict
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Output,
+)
+
+
+class CaseHOLDScenario(Scenario):
+    """
+     CaseHOLD QA
+       CaseHOLD is a multiple choice question answering task derived from legal citations in judicial rulings.
+       CaseHOLD consists of ~53,000 questions, mined from the Harvard Law Library case law corpus.
+
+     Dataset repository
+       https://huggingface.co/datasets/casehold/casehold
+     Publication
+       "When Does Pretraining Help? Assessing Self-Supervised Learning for Law and the CaseHOLD Dataset"
+       ICAIL, 2021
+       https://reglab.stanford.edu/data/casehold-benchmark/
+       https://arxiv.org/abs/2104.08671
+
+    Data content
+      The citing context from the judicial decision serves as the prompt for the question.
+      The answer choices are holding statements derived from citations following text in a legal decision.
+      There are five answer choices for each citing text.
+      The correct answer is the holding statement that corresponds to the citing text.
+      The four incorrect answers are other holding statements.
+
+    """  # noqa: E501
+
+    name = "casehold"
+    description = "CaseHOLD (Case Holdings On Legal Decisions) is a multiple choice question answering scenario where the task is to identify the relevant holding of a cited case [(Zheng et al, 2021)](https://arxiv.org/pdf/2104.08671.pdf)."  # noqa: E501
+    tags = ["question_answering", "legal"]
+
+    # Note: Skip the validation split since we don't need it
+    HELM_SPLIT_NAME_TO_DATASETS_SPLIT_NAME = {TRAIN_SPLIT: "train", TEST_SPLIT: "test"}
+    NUM_REFERENCES = 5
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path: str = os.path.join(output_path, "data")
+        ensure_directory_exists(data_path)
+        dataset: DatasetDict = load_dataset(
+            "casehold/casehold",
+            "all",
+            cache_dir=data_path,
+        )
+
+        instances: List[Instance] = []
+        for helm_split_name, datasets_split_name in self.HELM_SPLIT_NAME_TO_DATASETS_SPLIT_NAME.items():
+            split_data = dataset[datasets_split_name]
+            for example in split_data:
+                example_id = example["example_id"]
+                citing_prompt = example["citing_prompt"]
+                holdings = [example[f"holding_{i}"] for i in range(self.NUM_REFERENCES)]
+                correct_label: str = example["label"]
+                references = [
+                    Reference(Output(text=holdings[i]), tags=([CORRECT_TAG] if correct_label == str(i) else []))
+                    for i in range(self.NUM_REFERENCES)
+                ]
+                instance = Instance(
+                    input=Input(text=citing_prompt),
+                    references=references,
+                    split=helm_split_name,
+                    id=f"id{example_id}",
+                )
+                instances.append(instance)
+        return instances