crfm-helm 0.5.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/benchmark/run_specs/vlm_run_specs.py

@@ -10,7 +10,8 @@ from helm.benchmark.adaptation.adapters.adapter_factory import (
 from helm.benchmark.metrics.common_metric_specs import (
     get_basic_reference_metric_specs,
     get_exact_match_metric_specs,
-    get_open_ended_generation_metric_specs,
+    get_generative_harms_metric_specs,
+    get_basic_metric_specs,
 )
 from helm.benchmark.metrics.metric import MetricSpec
 from helm.benchmark.run_spec import RunSpec, run_spec_function
@@ -22,7 +23,7 @@ from helm.benchmark.annotation.annotator import AnnotatorSpec
 # Prototypical adapter specs for VLM evaluation
 
 
-def get_generation_adapter_spec(
+def _get_generation_adapter_spec(
     instructions: str = "",
     input_prefix: str = "",
     input_suffix: str = "",
@@ -45,18 +46,31 @@ def get_generation_adapter_spec(
         num_outputs=1,
         max_tokens=max_tokens,
         stop_sequences=stop_sequences if stop_sequences is not None else [],
+        temperature=0.0,
         random=None,
     )
 
 
-def get_short_answer_generation_adapter_spec():
-    return get_generation_adapter_spec(
-        instructions="Just give a short answer without answering in a complete sentence.",
+def _get_short_answer_generation_adapter_spec(instructions: Optional[str] = None) -> AdapterSpec:
+    return _get_generation_adapter_spec(
+        instructions=(
+            "Just give a short answer without answering in a complete sentence."
+            if instructions is None
+            else instructions
+        ),
         max_tokens=20,
     )
 
 
-def get_multiple_choice_joint_adapter_spec(
+def _get_captioning_adapter_spec() -> AdapterSpec:
+    return _get_generation_adapter_spec(
+        instructions="Generate a caption for the following image. The caption should be short and does "
+        "not need to be a complete sentence.",
+        max_tokens=20,
+    )
+
+
+def _get_multiple_choice_joint_adapter_spec(
     input_noun: Optional[str],
     output_noun: str,
     max_train_instances: int = 0,
@@ -84,7 +98,13 @@ def get_multiple_choice_joint_adapter_spec(
 # VHELM metric specs
 
 
-def get_image2structure_metric_specs(
+def _get_open_ended_generation_metric_specs() -> List[MetricSpec]:
+    return get_basic_metric_specs(
+        ["exact_match", "quasi_exact_match", "f1_score", "rouge_l", "bleu_1", "bleu_4", "cider"]
+    )
+
+
+def _get_image2structure_metric_specs(
     generation_type: str,
     metric_names: Optional[List[str]] = None,
     args: Optional[Dict] = None,
@@ -97,7 +117,9 @@ def get_image2structure_metric_specs(
         metric_names = [
             AnnotatedImageMetrics.PIXEL_SIMILARITY,
             AnnotatedImageMetrics.FID_SIMILARITY,
-            AnnotatedImageMetrics.EARTH_MOVER_SIMILARITY,
+            AnnotatedImageMetrics.BLOCK_EARTH_MOVER_SIMILARITY,
+            AnnotatedImageMetrics.BLOCK_EARTH_MOVER_SIMILARITY_NORM2,
+            AnnotatedImageMetrics.BLOCK_EARTH_MOVER_SIMILARITY_NORM1,
         ]
     if include_edit_similarity:
         metric_names.append(AnnotatedImageMetrics.EDIT_SIMILARITY)
@@ -121,13 +143,35 @@ def get_image2structure_metric_specs(
 # VHELM run specs
 
 
+@run_spec_function("a_okvqa")
+def get_a_okvqa_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.vision_language.a_okvqa_scenario.AOKVQAScenario",
+        args={},
+    )
+
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
+        input_noun=None, output_noun="Answer", max_train_instances=0
+    )
+
+    metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
+    run_spec_name: str = "a_okvqa"
+    return RunSpec(
+        name=run_spec_name,
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
 @run_spec_function("chart2csv")
 def get_chart2csv_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.vision_language.image2structure.chart2csv_scenario.Chart2CSVScenario",
         args={},
     )
-    adapter_spec: AdapterSpec = get_generation_adapter_spec(
+    adapter_spec: AdapterSpec = _get_generation_adapter_spec(
         instructions="Generate the CSV for the chart. Some of the labels may be missing due to the size of the chart. "
         "Please infer the missing labels based on the surrounding context. "
         "Just give the CSV without any explanation.",
@@ -145,15 +189,69 @@ def get_chart2csv_spec() -> RunSpec:
     )
 
 
+@run_spec_function("crossmodal_3600")
+def get_crossmodal_3600_spec(location: str, language: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.vision_language.crossmodal_3600_scenario.Crossmodal3600Scenario",
+        args={"location": location, "language": language},
+    )
+    adapter_spec: AdapterSpec = _get_generation_adapter_spec(max_tokens=20)
+    metric_specs: List[MetricSpec] = get_exact_match_metric_specs() + _get_open_ended_generation_metric_specs()
+
+    run_spec_name: str = "crossmodal_3600"
+    return RunSpec(
+        name=f"{run_spec_name}:location={location},language={language}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+@run_spec_function("flickr30k")
+def get_flickr30k_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.vision_language.flickr30k_scenario.Flickr30KScenario", args={}
+    )
+    adapter_spec: AdapterSpec = _get_captioning_adapter_spec()
+    metric_specs: List[MetricSpec] = get_exact_match_metric_specs() + _get_open_ended_generation_metric_specs()
+
+    run_spec_name: str = "flickr30k"
+    return RunSpec(
+        name=run_spec_name,
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+@run_spec_function("gqa")
+def get_gqa_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.vision_language.gqa_scenario.GQAScenario", args={}
+    )
+    adapter_spec: AdapterSpec = _get_short_answer_generation_adapter_spec(
+        instructions="Answer the question using a single word or phrase."
+    )
+    metric_specs: List[MetricSpec] = get_exact_match_metric_specs() + _get_open_ended_generation_metric_specs()
+
+    run_spec_name: str = "gqa"
+    return RunSpec(
+        name=run_spec_name,
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
 @run_spec_function("hateful_memes")
 def get_hateful_memes_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.vision_language.hateful_memes_scenario.HatefulMemesScenario", args={}
     )
-    adapter_spec: AdapterSpec = get_generation_adapter_spec(
-        instructions="Answer Yes or No without an explanation.",
-        max_tokens=3,
-    )
+    adapter_spec = _get_multiple_choice_joint_adapter_spec(input_noun=None, output_noun="Answer", max_train_instances=0)
     metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
 
     run_spec_name: str = "hateful_memes"
@@ -166,13 +264,111 @@ def get_hateful_memes_spec() -> RunSpec:
     )
 
 
+@run_spec_function("mm_safety_bench")
+def get_mm_safety_bench_spec(subset: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.vision_language.mm_safety_bench_scenario.MMSafetyBenchScenario",
+        args={"subset": subset},
+    )
+    adapter_spec: AdapterSpec = _get_generation_adapter_spec(max_tokens=500)
+    metric_specs: List[MetricSpec] = get_generative_harms_metric_specs(
+        include_basic_metrics=True, include_generative_harms_metrics=True
+    )
+
+    run_spec_name: str = "mm_safety_bench"
+    return RunSpec(
+        name=f"{run_spec_name}:subset={subset}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+@run_spec_function("mscoco_captioning")
+def get_mscoco_captioning_spec(long: bool = False) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.vision_language.mscoco_captioning_scenario.MSCOCOCaptioningScenario",
+        args={},
+    )
+
+    adapter_spec: AdapterSpec
+    if long:
+        adapter_spec = _get_generation_adapter_spec(
+            instructions="Generate a long, detailed caption for the following image.",
+            max_tokens=150,
+        )
+    else:
+        adapter_spec = _get_captioning_adapter_spec()
+    metric_specs: List[MetricSpec] = get_exact_match_metric_specs() + _get_open_ended_generation_metric_specs()
+
+    run_spec_name: str = "mscoco_captioning"
+    if long:
+        run_spec_name += "_long"
+
+    return RunSpec(
+        name=run_spec_name,
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+@run_spec_function("mscoco_categorization")
+def get_mscoco_categorization_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.vision_language.mscoco_categorization_scenario."
+        "MSCOCOCategorizationScenario",
+        args={},
+    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
+        input_noun=None, output_noun="Answer", max_train_instances=0
+    )
+
+    metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
+
+    run_spec_name: str = "mscoco_categorization"
+    return RunSpec(
+        name=run_spec_name,
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+@run_spec_function("originality_vlm")
+def get_originality_vlm_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.vision_language.originality_scenario.OriginalityScenario", args={}
+    )
+    adapter_spec: AdapterSpec = _get_generation_adapter_spec(max_tokens=500)
+    metric_specs: List[MetricSpec] = get_generative_harms_metric_specs(
+        include_basic_metrics=True, include_generative_harms_metrics=True
+    )
+
+    run_spec_name: str = "originality_vlm"
+    return RunSpec(
+        name=run_spec_name,
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
 @run_spec_function("viz_wiz")
 def get_viz_wiz_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.vision_language.viz_wiz_scenario.VizWizScenario", args={}
     )
-    adapter_spec: AdapterSpec = get_short_answer_generation_adapter_spec()
-    metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
+    adapter_spec: AdapterSpec = _get_short_answer_generation_adapter_spec(
+        # Following https://arxiv.org/abs/2310.03744
+        instructions="When the provided information is insufficient, respond with 'Unanswerable'. "
+        "Answer the question using a single word or phrase."
+    )
+    metric_specs: List[MetricSpec] = get_exact_match_metric_specs() + _get_open_ended_generation_metric_specs()
 
     run_spec_name: str = "viz_wiz"
     return RunSpec(
@@ -189,8 +385,12 @@ def get_vqa_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.vision_language.vqa_scenario.VQAScenario", args={}
     )
-    adapter_spec: AdapterSpec = get_short_answer_generation_adapter_spec()
-    metric_specs: List[MetricSpec] = get_exact_match_metric_specs() + get_open_ended_generation_metric_specs()
+    # Following https://arxiv.org/abs/2310.03744
+    adapter_spec: AdapterSpec = _get_short_answer_generation_adapter_spec(
+        instructions='Answer the question using a single word or phrase. When the question asks "How many...", '
+        "respond with just a number (e.g., 3) and not the word corresponding to the number."
+    )
+    metric_specs: List[MetricSpec] = get_exact_match_metric_specs() + _get_open_ended_generation_metric_specs()
 
     run_spec_name: str = "vqa"
     return RunSpec(
@@ -208,11 +408,11 @@ def get_image2latex_spec(subset: str, recompile_prompt: bool = False, args: Opti
         class_name="helm.benchmark.scenarios.vision_language.image2structure.latex_scenario.LatexScenario",
         args={"subset": subset, "recompile_prompt": recompile_prompt},
     )
-    adapter_spec: AdapterSpec = get_generation_adapter_spec(
+    adapter_spec: AdapterSpec = _get_generation_adapter_spec(
         instructions="Just give a short answer without answering in a complete sentence.",
         max_tokens=2000,
     )
-    metric_specs: List[MetricSpec] = get_image2structure_metric_specs(
+    metric_specs: List[MetricSpec] = _get_image2structure_metric_specs(
         generation_type="latex",
         args=args,
         include_edit_similarity=True,
@@ -241,11 +441,11 @@ def get_image2webpage_spec(subset: str, recompile_prompt: bool = False, args: Op
         class_name="helm.benchmark.scenarios.vision_language.image2structure.webpage_scenario.WebpageScenario",
         args={"subset": subset, "recompile_prompt": recompile_prompt},
     )
-    adapter_spec: AdapterSpec = get_generation_adapter_spec(
+    adapter_spec: AdapterSpec = _get_generation_adapter_spec(
         instructions="Just give a short answer without answering in a complete sentence.",
         max_tokens=2000,
     )
-    metric_specs: List[MetricSpec] = get_image2structure_metric_specs(
+    metric_specs: List[MetricSpec] = _get_image2structure_metric_specs(
         generation_type="webpage",
         args=args,
         include_edit_similarity=True,
@@ -268,17 +468,45 @@ def get_image2webpage_spec(subset: str, recompile_prompt: bool = False, args: Op
     )
 
 
+@run_spec_function("math_vista")
+def get_math_vista_spec(grade: str, question_type: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.vision_language.math_vista_scenario.MathVistaScenario",
+        args={"grade": grade, "question_type": question_type},
+    )
+
+    adapter_spec: AdapterSpec
+    if question_type == "free_form":
+        adapter_spec = _get_short_answer_generation_adapter_spec()
+    elif question_type == "multi_choice":
+        adapter_spec = _get_multiple_choice_joint_adapter_spec(
+            input_noun=None, output_noun="Answer", max_train_instances=0
+        )
+    else:
+        raise ValueError(f"Invalid question type: {question_type}")
+
+    metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
+    run_spec_name: str = "math_vista"
+    return RunSpec(
+        name=f"{run_spec_name}:grade={grade},question_type={question_type}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
 @run_spec_function("image2musicsheet")
 def get_image2musicsheet_spec(args: Optional[Dict] = None) -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.vision_language.image2structure.musicsheet_scenario.MusicSheetScenario",
         args={"subset": "music", "recompile_prompt": False},  # There os only one subset for music sheets
     )
-    adapter_spec: AdapterSpec = get_generation_adapter_spec(
+    adapter_spec: AdapterSpec = _get_generation_adapter_spec(
         instructions="Just give a short answer without answering in a complete sentence.",
         max_tokens=2000,
     )
-    metric_specs: List[MetricSpec] = get_image2structure_metric_specs(
+    metric_specs: List[MetricSpec] = _get_image2structure_metric_specs(
         generation_type="lilypond",
         args=args,
         include_edit_similarity=False,  # No ground truth for music sheets
@@ -292,7 +520,7 @@ def get_image2musicsheet_spec(args: Optional[Dict] = None) -> RunSpec:
 
     run_spec_name: str = "image2musicsheet"
     return RunSpec(
-        name=f"{run_spec_name}",
+        name=run_spec_name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=metric_specs,
@@ -310,10 +538,14 @@ def get_mmmu_spec(subject: str, question_type: str) -> RunSpec:
 
     adapter_spec: AdapterSpec
     if question_type == "open":
-        adapter_spec = get_short_answer_generation_adapter_spec()
+        adapter_spec = _get_short_answer_generation_adapter_spec()
     elif question_type == "multiple-choice":
-        adapter_spec = get_multiple_choice_joint_adapter_spec(
-            input_noun=None, output_noun="Answer", max_train_instances=0
+        adapter_spec = _get_multiple_choice_joint_adapter_spec(
+            input_noun=None,
+            output_noun="Answer",
+            max_train_instances=0,
+            # instructions="Refer to the figure(s) and answer the multiple choice question by responding with just "
+            # "the letter of the correct answer (e.g., A, B, C, D, E).",
         )
     else:
         raise ValueError(f"Invalid question type: {question_type}")
@@ -335,7 +567,7 @@ def get_unicorn_spec(subject: str) -> RunSpec:
         class_name="helm.benchmark.scenarios.vision_language.unicorn_scenario.UnicornScenario",
         args={"subject": subject},
     )
-    adapter_spec: AdapterSpec = get_generation_adapter_spec(
+    adapter_spec: AdapterSpec = _get_generation_adapter_spec(
         instructions="Only give numerical or boolean answer without an explanation."
     )
     metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
@@ -355,8 +587,8 @@ def get_bingo_spec(subject: str) -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.vision_language.bingo_scenario.BingoScenario", args={"subject": subject}
     )
-    adapter_spec: AdapterSpec = get_short_answer_generation_adapter_spec()
-    metric_specs: List[MetricSpec] = get_open_ended_generation_metric_specs()
+    adapter_spec: AdapterSpec = _get_short_answer_generation_adapter_spec()
+    metric_specs: List[MetricSpec] = _get_open_ended_generation_metric_specs()
 
     run_spec_name: str = "bingo"
     return RunSpec(
@@ -377,9 +609,9 @@ def get_multipanelvqa_spec(subject: str, question_type: str) -> RunSpec:
 
     adapter_spec: AdapterSpec
     if question_type == "open":
-        adapter_spec = get_short_answer_generation_adapter_spec()
+        adapter_spec = _get_short_answer_generation_adapter_spec()
     elif question_type == "multiple-choice":
-        adapter_spec = get_multiple_choice_joint_adapter_spec(
+        adapter_spec = _get_multiple_choice_joint_adapter_spec(
             input_noun=None, output_noun="Answer", max_train_instances=0
         )
     else:
@@ -401,7 +633,7 @@ def get_pope_spec() -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.vision_language.pope_scenario.POPEScenario",
     )
-    adapter_spec: AdapterSpec = get_multiple_choice_joint_adapter_spec(
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
         input_noun=None, output_noun="Answer", max_train_instances=0
     )
     metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
@@ -422,7 +654,7 @@ def get_seed_bench_spec(subject: str) -> RunSpec:
         class_name="helm.benchmark.scenarios.vision_language.seed_bench_scenario.SEEDBenchScenario",
         args={"subject": subject},
     )
-    adapter_spec: AdapterSpec = get_multiple_choice_joint_adapter_spec(
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
         input_noun=None, output_noun="Answer", max_train_instances=0
     )
     metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
@@ -443,7 +675,7 @@ def get_mme_spec(subject: str) -> RunSpec:
         class_name="helm.benchmark.scenarios.vision_language.mme_scenario.MMEScenario",
         args={"subject": subject},
     )
-    adapter_spec: AdapterSpec = get_multiple_choice_joint_adapter_spec(
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
         input_noun=None, output_noun="Answer", max_train_instances=0
     )
     metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
@@ -464,7 +696,7 @@ def get_heim_human_eval_spec(question_type: str) -> RunSpec:
         class_name="helm.benchmark.scenarios.vision_language.heim_human_eval_scenario.HEIMHumanEvalScenario",
         args={"question_type": question_type},
     )
-    adapter_spec: AdapterSpec = get_multiple_choice_joint_adapter_spec(
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
         input_noun=None,
         output_noun="Answer",
         num_outputs=1,
@@ -482,14 +714,38 @@ def get_heim_human_eval_spec(question_type: str) -> RunSpec:
     )
 
 
+@run_spec_function("pairs")
+def get_pairs_spec(subset: str, person: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.vision_language.pairs_scenario.PAIRSScenario",
+        args={"subset": subset, "person": person},
+    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
+        input_noun=None,
+        output_noun="Answer",
+        num_outputs=1,
+        max_train_instances=0,
+    )
+    metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
+
+    run_spec_name: str = "pairs"
+    return RunSpec(
+        name=f"{run_spec_name}:subset={subset},person={person}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
 @run_spec_function("mementos")
 def get_mementos_spec(subject: str) -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.vision_language.mementos_scenario.MementosScenario",
         args={"subject": subject},
     )
-    adapter_spec: AdapterSpec = get_short_answer_generation_adapter_spec()
-    metric_specs: List[MetricSpec] = get_open_ended_generation_metric_specs()
+    adapter_spec: AdapterSpec = _get_short_answer_generation_adapter_spec()
+    metric_specs: List[MetricSpec] = _get_open_ended_generation_metric_specs()
 
     run_spec_name: str = "mementos"
     return RunSpec(

helm/benchmark/scenarios/legalbench_scenario.py

@@ -96,8 +96,12 @@ class LegalBenchScenario(Scenario):
 
         # Download data from Huggingface. LegalBench provides splits for samples to
         # be used for prompt construction and for testing.
-        train_dataset = datasets.load_dataset("nguha/legalbench", self.subset, cache_dir=cache_dir, split="train")
-        test_dataset = datasets.load_dataset("nguha/legalbench", self.subset, cache_dir=cache_dir, split="test")
+        train_dataset = datasets.load_dataset(
+            "nguha/legalbench", self.subset, trust_remote_code=True, cache_dir=cache_dir, split="train"
+        )
+        test_dataset = datasets.load_dataset(
+            "nguha/legalbench", self.subset, trust_remote_code=True, cache_dir=cache_dir, split="test"
+        )
         assert isinstance(train_dataset, datasets.Dataset)
         assert isinstance(test_dataset, datasets.Dataset)
 

helm/benchmark/scenarios/math_scenario.py

@@ -368,7 +368,7 @@ class MATHScenario(Scenario):
         cache_dir = os.path.join(output_path, "data")
         ensure_directory_exists(cache_dir)
         data = (
-            typing.cast(DatasetDict, load_dataset("competition_math", cache_dir=cache_dir))
+            typing.cast(DatasetDict, load_dataset("competition_math", trust_remote_code=True, cache_dir=cache_dir))
             .sort("problem")
             .shuffle(seed=42)
         )

helm/benchmark/scenarios/vision_language/a_okvqa_scenario.py

@@ -0,0 +1,83 @@
+import os
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    ALL_SPLITS,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+
+
+class AOKVQAScenario(Scenario):
+    """
+    A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of commonsense
+    and world knowledge to answer.
+
+    @misc{schwenk2022aokvqa,
+          title={A-OKVQA: A Benchmark for Visual Question Answering using World Knowledge},
+          author={Dustin Schwenk and Apoorv Khandelwal and Christopher Clark and Kenneth Marino and Roozbeh Mottaghi},
+          year={2022},
+          eprint={2206.01718},
+          archivePrefix={arXiv},
+          primaryClass={cs.CV}
+    }
+
+    Paper: https://arxiv.org/abs/2206.01718
+    Website: https://huggingface.co/datasets/HuggingFaceM4/A-OKVQA
+    """
+
+    HF_DATASET_NAME: str = "HuggingFaceM4/A-OKVQA"
+
+    name = "a_okvqa"
+    description = (
+        "A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of "
+        "commonsense and world knowledge to answer ([paper](https://arxiv.org/abs/2206.01718))."
+    )
+    tags = ["vision-language", "knowledge", "reasoning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        instances: List[Instance] = []
+        for helm_split in ALL_SPLITS:
+            if helm_split == TEST_SPLIT:
+                # The examples in the test split does not have answers
+                continue
+
+            split = "validation" if helm_split == VALID_SPLIT else helm_split
+
+            for row in tqdm(load_dataset(self.HF_DATASET_NAME, cache_dir=output_path, split=split)):
+                image_filename: str = f"{row['question_id']}.jpg"
+                local_image_path: str = os.path.join(images_path, image_filename)
+                image = row["image"]
+                if not os.path.exists(local_image_path):
+                    image.save(local_image_path)
+
+                content: List[MediaObject] = [
+                    MediaObject(location=local_image_path, content_type="image/jpeg"),
+                    MediaObject(text=row["question"], content_type="text/plain"),
+                ]
+                instances.append(
+                    Instance(
+                        Input(multimedia_content=MultimediaObject(content)),
+                        references=[
+                            Reference(Output(text=choice), tags=[CORRECT_TAG] if i == row["correct_choice_idx"] else [])
+                            for i, choice in enumerate(row["choices"])
+                        ],
+                        split=helm_split,
+                    )
+                )
+
+        return instances