crfm-helm 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/run_specs/bluex_run_specs.py

@@ -0,0 +1,40 @@
+from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("bluex")
+def get_bluex_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bluex_scenario.BLUEXScenario", args={})
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="""
+        Escolha a alternativa correta para as questões de vestibulares (responda apenas com a letra).
+        Exemplo de Pergunta com a resposta:
+        Em um romance narrado em primeira pessoa, o narrador participa dos acontecimentos da trama,
+        relatando suas próprias experiências e sentimentos. Qual alternativa apresenta essa característica?
+
+        (A) Narrador onisciente que conhece os pensamentos de todas as personagens.
+        (B) Narrador que descreve os fatos de forma imparcial, sem envolvimento emocional.
+        (C) Narrador-personagem que vivencia e relata os eventos da história.
+        (D) Narrador observador que apenas registra as ações visíveis.
+        (E) Narrador em segunda pessoa que se dirige constantemente ao leitor.
+
+        Resposta correta: C
+
+        A partir disso, responda:
+        """,
+        input_noun="Pergunta",
+        output_noun="Resposta",
+    )
+
+    return RunSpec(
+        name="bluex",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["bluex"],
+    )

helm/benchmark/run_specs/classic_run_specs.py

@@ -35,7 +35,6 @@ from helm.benchmark.metrics.common_metric_specs import (
     get_f1_metric_specs,
     get_generative_harms_metric_specs,
     get_language_modeling_metric_specs,
-    get_numeracy_metric_specs,
     get_open_ended_generation_metric_specs,
     get_summarization_metric_specs,
     get_basic_generation_metric_specs,
@@ -381,58 +380,6 @@ def get_raft_spec(subset: str) -> RunSpec:
     )
 
 
-@run_spec_function("numeracy")
-def get_numeracy_spec(
-    relation_type: str = "linear", mode: str = "function", seed: str = "0", run_solver: str = "False"
-) -> RunSpec:
-    from helm.benchmark.scenarios.numeracy_scenario import get_numeracy_adapter_spec, RELTYPE_INFO
-
-    run_solver_bool: bool = True if run_solver.lower() == "true" else False
-    del run_solver
-    random_seed = int(seed)
-    scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.numeracy_scenario.NumeracyScenario",
-        args={"seed": random_seed, "relation_type": relation_type, "mode": mode},
-    )
-
-    if mode in ["example", "standard"]:
-        # Test a model's ability to impute datapoints for a given (example or randomly sampled) relation.
-        adapter_args: Dict[str, Any] = {
-            "max_train_instances": 100,
-            "max_eval_instances": 100,
-            "dim": RELTYPE_INFO[relation_type].num_variables + 1,
-        }
-    elif mode == "function":
-        # Test a model's ability to impute datapoints for randomly sampled relations
-        # (resampled for each evaluation point).
-        adapter_args = {
-            "instructions": "",
-            "max_train_instances": 0,  # Turn off general version of `function` mode because it doesn't cleanly
-            # capture a higher-order version of this task / is a little convoluted
-            # for models, currently.
-            # (In the general version, the model sees other relations of the same class,
-            # and needs to impute a datapoint for the last one. Presumably, inferring
-            # the class - eg. the degree of the relation - would help.)
-            "max_eval_instances": 1000,
-            "dim": RELTYPE_INFO[relation_type].num_variables + 1,
-            "instance_prefix": "\n\n",
-        }
-    else:
-        raise ValueError(f"Invalid mode: {mode}")
-
-    adapter_spec = get_numeracy_adapter_spec(**adapter_args)  # Construct the AdapterSpec using a helper function.
-    # `get_numeracy_adapter_spec` is defined in numeracy_scenario.py
-    # because it is used within the scenario to construct the instances themselves.
-
-    return RunSpec(
-        name=f"numeracy:relation_type={relation_type},mode={mode}",
-        scenario_spec=scenario_spec,
-        adapter_spec=adapter_spec,
-        metric_specs=get_numeracy_metric_specs(run_solver_bool),
-        groups=["numeracy"],
-    )
-
-
 @run_spec_function("boolq")
 def get_boolq_spec(only_contrast=False) -> RunSpec:
     scenario_spec = ScenarioSpec(
@@ -806,12 +753,12 @@ def get_xsum_sampled_summarization_spec(temperature: float = 0.3, device: str =
     )
 
     return RunSpec(
-        name=f"summarization_xsum:temperature={temperature},device={device}",
+        name=f"summarization_xsum_sampled:temperature={temperature},device={device}",
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_summarization_metric_specs({"task": "summarization_xsum_sampled", "device": device})
         + get_generative_harms_metric_specs(),
-        groups=["summarization_xsum"],
+        groups=["summarization_xsum_sampled"],
     )
 
 

helm/benchmark/run_specs/codeinsights_run_specs.py

@@ -0,0 +1,192 @@
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+from helm.benchmark.adaptation.common_adapter_specs import get_generation_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_basic_metric_specs
+from helm.benchmark.metrics.codeinsights_metric_specs import (
+    get_functional_correctness_metric_specs,
+    get_comprehensive_code_evaluation_metric_specs,
+    get_edge_case_metric_specs,
+    get_code_efficiency_metric_specs,
+)
+
+
+@run_spec_function("codeinsights_correct_code")
+def get_codeinsights_correct_code_run_spec(tpr: float = 0.0, num_testcases: int = 1) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.codeinsights_correct_code_scenario.CodeInsightsCorrectCodeScenario",
+        args={"num_testcases": num_testcases},
+    )
+
+    instruction = (
+        "You are a skilled C++ programmer working on a foundational programming course assignment. "
+        "Your task is to write correct, efficient C++ code that solves the given problem. "
+        "Write clean, well-structured code that follows good programming practices. "
+        "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template."
+        "DO NOT reproduce the template part as the generated code would be inserted to the template,"
+        "and make sure the code is compatible with the Unit Test Input"
+        "Ensure your code is correct, efficient, includes any class definition when needed, and handles all edge cases properly."
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=instruction,
+        output_noun="Your code",
+        stop_sequences=[],
+        max_tokens=4000,
+        temperature=tpr,
+    )
+
+    return RunSpec(
+        name=f"codeinsights_correct_code:temperature={adapter_spec.temperature},num_testcases={num_testcases}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_functional_correctness_metric_specs() + get_basic_metric_specs([]),
+        groups=["codeinsights", "codeinsights_correct_code"],
+    )
+
+
+@run_spec_function("codeinsights_student_coding")
+def get_codeinsights_student_coding_run_spec(tpr: float = 0.0, num_testcases: int = 1) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.codeinsights_student_coding_scenario.CodeInsightsStudentCodingScenario",
+        args={"num_testcases": num_testcases},
+    )
+
+    instruction = (
+        "You are the same student who wrote the three examples below in your foundational C++ course. "
+        "Mimic exactly your personal coding style, conventions, and level of proficiency—"
+        "do not over‐optimize or introduce unfamiliar patterns. "
+        "Include the same sort of formatting, variable names, and minor imperfections you demonstrated. "
+        "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template."
+        "DO NOT reproduce the template part as the generated code would be inserted to the template,"
+        "and make sure the code is compatible with the Unit Test Input"
+        "Ensure your code includes any class definition when needed."
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=instruction,
+        output_noun="Your code",
+        stop_sequences=[],
+        max_tokens=4000,
+        temperature=tpr,
+    )
+
+    return RunSpec(
+        name=f"codeinsights_student_coding:temperature={adapter_spec.temperature},num_testcases={num_testcases}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_comprehensive_code_evaluation_metric_specs() + get_basic_metric_specs([]),
+        groups=["codeinsights", "codeinsights_student_coding"],
+    )
+
+
+@run_spec_function("codeinsights_student_mistake")
+def get_codeinsights_student_mistake_run_spec(tpr: float = 0.0, num_testcases: int = 1) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.codeinsights_student_mistake_scenario.CodeInsightsStudentMistakeScenario",
+        args={"num_testcases": num_testcases},
+    )
+
+    instruction = (
+        "You are a C++ student with a consistent personal style, conventions, and proficiency level.\n"
+        "Your task is to attempt the target problem **but introduce realistic mistake** you would typically make—"
+        "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template."
+        "DO NOT reproduce the template part as the generated code would be inserted to the template,"
+        "and make sure the code is compatible with the Unit Test Input"
+        "Ensure your code is includes any class definition when needed."
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=instruction,
+        output_noun="Your code",
+        stop_sequences=[],
+        max_tokens=4000,
+        temperature=tpr,
+    )
+
+    return RunSpec(
+        name=f"codeinsights_student_mistake:temperature={adapter_spec.temperature},num_testcases={num_testcases}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_comprehensive_code_evaluation_metric_specs() + get_basic_metric_specs([]),
+        groups=["codeinsights", "codeinsights_student_mistake"],
+    )
+
+
+@run_spec_function("codeinsights_code_efficiency")
+def get_codeinsights_code_efficiency_run_spec(tpr: float = 0.0, num_testcases: int = 1) -> RunSpec:
+    """
+    Run specification for code efficiency evaluation scenario.
+
+    This scenario evaluates whether LLM-generated code has similar runtime efficiency
+    as the original student code. It focuses on problems where both solutions are
+    functionally correct and measures runtime performance alignment.
+
+    Requires C++ compiler (g++) to be available for actual compilation and execution.
+    """
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.codeinsights_code_efficiency_scenario.CodeInsightsCodeEfficiencyScenario",
+        args={"num_testcases": num_testcases},
+    )
+
+    instruction = (
+        "You are the same student who wrote the three examples below in your foundational C++ course. "
+        "Mimic exactly your personal coding style, conventions, and make sure to generate a correct code. "
+        "Do not over-optimize or introduce unfamiliar patterns. If the code is correct but inefficient, "
+        "imitate the inefficiency. "
+        "If the student writes efficiently, write efficiently too. "
+        "Include the same sort of formatting, variable names, and minor imperfections you demonstrated. "
+        "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template."
+        "DO NOT reproduce the template part as the generated code would be inserted to the template,"
+        "and make sure the code is compatible with the Unit Test Input"
+        "Ensure your code is correct, includes any class definition when needed, and handles all edge cases properly."
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=instruction,
+        output_noun="Your code",
+        stop_sequences=[],
+        max_tokens=4000,
+        temperature=tpr,
+    )
+
+    return RunSpec(
+        name=f"codeinsights_code_efficiency:temperature={adapter_spec.temperature},num_testcases={num_testcases}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_code_efficiency_metric_specs(
+            num_runtime_runs=5,  # Run each solution 5 times for averaging
+            timeout_seconds=10,  # 10 second timeout per execution
+        )
+        + get_basic_metric_specs([]),
+        groups=["codeinsights", "codeinsights_code_efficiency"],
+    )
+
+
+@run_spec_function("codeinsights_edge_case")
+def get_codeinsights_edge_case_run_spec(tpr: float = 0.0, num_testcases: int = 1) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.codeinsights_edge_case_scenario.CodeInsightsEdgeCaseScenario",
+        args={"num_testcases": num_testcases},
+    )
+
+    instruction = (
+        "You are a student studying C++ with a consistent personal style, conventions, and proficiency level.\n"
+        "Your task is to identify which test case you would likely to fail for a given question with unit tests.\n"
+        "Respond only with integer of the unittest number\n\n"
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=instruction,
+        output_noun="Your code",
+        stop_sequences=[],
+        max_tokens=4000,
+        temperature=tpr,
+    )
+
+    return RunSpec(
+        name=f"codeinsights_edge_case:temperature={adapter_spec.temperature},num_testcases={num_testcases}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_edge_case_metric_specs() + get_basic_metric_specs([]),
+        groups=["codeinsights", "codeinsights_edge_case"],
+    )

helm/benchmark/run_specs/healthqa_br_run_specs.py

@@ -0,0 +1,40 @@
+from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("healthqa_br")
+def get_healthqa_br_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.healthqa_br_scenario.HEALTHQA_BR_Scenario", args={}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="""
+        Escolha a alternativa correta para as questões de medicina (responda apenas com a letra).
+        Exemplo de Pergunta com a resposta:
+        Qual dos seguintes órgãos é responsável pela produção da insulina no corpo humano?
+        A) Fígado
+        B) Rins
+        C) Pâncreas
+        D) Baço
+        E) Coração
+
+        Resposta correta: C
+
+        A partir disso, responda:
+        """,
+        input_noun="Pergunta",
+        output_noun="Resposta",
+    )
+
+    return RunSpec(
+        name="healthqa_br",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["healthqa_br"],
+    )

helm/benchmark/run_specs/heim_run_specs.py

@@ -60,7 +60,9 @@ def get_core_heim_metric_specs() -> List[MetricSpec]:
             class_name="helm.benchmark.metrics.image_generation.fractal_dimension_metric.FractalDimensionMetric",
             args={},
         ),
-        MetricSpec(class_name="helm.benchmark.metrics.image_generation.nsfw_metrics.NSFWMetric", args={}),
+        # Disabled due to keras issue.
+        # See: https://github.com/stanford-crfm/helm/issues/3741#issuecomment-3109478877
+        # MetricSpec(class_name="helm.benchmark.metrics.image_generation.nsfw_metrics.NSFWMetric", args={}),
         MetricSpec(class_name="helm.benchmark.metrics.image_generation.nudity_metrics.NudityMetric", args={}),
         MetricSpec(class_name="helm.benchmark.metrics.image_generation.watermark_metrics.WatermarkMetric", args={}),
     ] + get_basic_metric_specs(names=[])

helm/benchmark/run_specs/lmkt_run_specs.py

@@ -0,0 +1,144 @@
+"""Run spec functions for Vietnam WVS cultural alignment evaluation."""
+
+from helm.benchmark.adaptation.common_adapter_specs import (
+    get_generation_adapter_spec,
+)
+from helm.benchmark.metrics.common_metric_specs import (
+    get_exact_match_metric_specs,
+    get_f1_metric_specs,
+    get_open_ended_generation_metric_specs,
+)
+from helm.benchmark.metrics.lmkt_metric_specs import get_semantic_similarity_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+INSTRUCTIONS = {
+    "cultural_value_understanding_wvs": {
+        "en": {
+            "instructions": "Please respond as the {country} persona described below.",
+            "input_noun": "Question",
+            "output_noun": "Answer",
+        },
+        "vi": {
+            "instructions": "Vui lòng trả lời như một người {country} được mô tả bên dưới.",
+            "input_noun": "Câu hỏi",
+            "output_noun": "Trả lời",
+        },
+    },
+    "social_norm_application_normad": {
+        "en": {
+            "instructions": "Please respond as the {country} persona described below.",
+            "input_noun": "Situation",
+            "output_noun": "Response",
+        },
+        "vi": {
+            "instructions": "Vui lòng trả lời như một người {country} được mô tả bên dưới.",
+            "input_noun": "Tình huống",
+            "output_noun": "Phản hồi",
+        },
+    },
+    "social_norm_explanation_normad": {
+        "en": {
+            "instructions": "Please respond as the {country} persona described below.",
+            "input_noun": "Situation",
+            "output_noun": "Explanation",
+        },
+        "vi": {
+            "instructions": "Vui lòng trả lời như một người {country} được mô tả bên dưới.",
+            "input_noun": "Tình huống",
+            "output_noun": "Giải thích",
+        },
+    },
+}
+
+COUNTRIES = {
+    "US": "United States",
+    "VN": "Vietnam",
+}
+
+
+@run_spec_function("cultural_value_understanding_wvs")
+def get_cultural_value_understanding_wvs_spec(language: str, country: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.lmkt_scenarios.CulturalValueUnderstandingWVSScenario",
+        args={
+            "language": language,
+            "num_personas": 300,
+            "num_question_variants": 4,
+            "include_few_shot_examples": True,
+        },
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=INSTRUCTIONS["cultural_value_understanding_wvs"][language]["instructions"].format(
+            country=COUNTRIES[country]
+        ),
+        input_noun=INSTRUCTIONS["cultural_value_understanding_wvs"][language]["input_noun"],
+        output_noun=INSTRUCTIONS["cultural_value_understanding_wvs"][language]["output_noun"],
+        max_tokens=3,
+        stop_sequences=[],
+    )
+
+    return RunSpec(
+        name="cultural_value_understanding_wvs",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs() + get_f1_metric_specs(),
+        groups=["lmkt", "cultural_value_understanding_wvs"],
+    )
+
+
+@run_spec_function("social_norm_application_normad")
+def get_social_norm_application_normad_spec(language: str, country: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.lmkt_scenarios.SocialNormApplicationNormADScenario",
+        args={
+            "language": language,
+        },
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=INSTRUCTIONS["social_norm_application_normad"][language]["instructions"].format(
+            country=COUNTRIES[country]
+        ),
+        input_noun=INSTRUCTIONS["social_norm_application_normad"][language]["input_noun"],
+        output_noun=INSTRUCTIONS["social_norm_application_normad"][language]["output_noun"],
+        max_tokens=5,
+        stop_sequences=[],
+    )
+
+    return RunSpec(
+        name="social_norm_application_normad",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs() + get_f1_metric_specs(),
+        groups=["lmkt", "social_norm_application_normad"],
+    )
+
+
+@run_spec_function("social_norm_explanation_normad")
+def get_social_norm_explanation_normad_spec(language: str, country: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.lmkt_scenarios.SocialNormExplanationNormADScenario",
+        args={
+            "language": language,
+        },
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=INSTRUCTIONS["social_norm_explanation_normad"][language]["instructions"].format(
+            country=COUNTRIES[country]
+        ),
+        input_noun=INSTRUCTIONS["social_norm_explanation_normad"][language]["input_noun"],
+        output_noun=INSTRUCTIONS["social_norm_explanation_normad"][language]["output_noun"],
+        max_tokens=128,
+        stop_sequences=[],
+    )
+
+    return RunSpec(
+        name="social_norm_explanation_normad",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_open_ended_generation_metric_specs() + get_semantic_similarity_metric_specs(),
+        groups=["lmkt", "social_norm_explanation_normad"],
+    )

helm/benchmark/run_specs/long_context_run_specs.py

@@ -1,4 +1,9 @@
-from helm.benchmark.adaptation.adapter_spec import ADAPT_CHAT, ADAPT_GENERATION, AdapterSpec
+from helm.benchmark.adaptation.adapter_spec import (
+    ADAPT_CHAT,
+    ADAPT_GENERATION,
+    ADAPT_MULTIPLE_CHOICE_JOINT,
+    AdapterSpec,
+)
 from helm.benchmark.metrics.common_metric_specs import (
     get_exact_match_metric_specs,
     get_open_ended_generation_metric_specs,
@@ -29,6 +34,27 @@ def _get_long_context_generation_adapter_spec(max_tokens: int) -> AdapterSpec:
     )
 
 
+def _get_long_context_multiple_choice_adapter_spec(max_tokens: int) -> AdapterSpec:
+    return AdapterSpec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        global_prefix="",
+        global_suffix="",
+        instructions="Read the passage and answer the following question. Respond with only a single letter corresponding to your choice. Do not include a period in your answer.\n\n",  # noqa: E501
+        input_prefix="",
+        input_suffix="\n",
+        reference_prefix="A. ",
+        reference_suffix="\n",
+        output_prefix="\nAnswer the question above based on the passage. Respond with only a single letter corresponding to your choice. Do not include a period in your answer.\n",  # noqa: E501
+        output_suffix="",
+        instance_prefix="",
+        max_train_instances=0,
+        num_outputs=1,
+        temperature=0.0,
+        max_tokens=max_tokens,
+        stop_sequences=[],
+    )
+
+
 @run_spec_function("ruler_hotpotqa")
 def get_ruler_hotpotqa_spec(max_num_words: int = 131072) -> RunSpec:
     scenario_spec = ScenarioSpec(
@@ -96,6 +122,27 @@ def get_infinite_bench_en_qa_spec(max_num_words: int = 131072) -> RunSpec:
     )
 
 
+@run_spec_function("infinite_bench_en_mc")
+def get_infinite_bench_en_mc_spec(max_num_words: int = 131072) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.infinite_bench_en_mc_scenario.InfiniteBenchEnMCScenario",
+        args={
+            "max_num_words": max_num_words,
+        },
+    )
+
+    adapter_spec = _get_long_context_multiple_choice_adapter_spec(max_tokens=40)
+    metric_specs = get_exact_match_metric_specs()
+
+    return RunSpec(
+        name=f"infinite_bench_en_mc:max_num_words={max_num_words}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=["infinite_bench_en_mc"],
+    )
+
+
 @run_spec_function("infinite_bench_en_sum")
 def get_infinite_bench_en_sum_spec(max_num_words: int = 131072) -> RunSpec: