crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/run.py

@@ -9,7 +9,7 @@ from helm.benchmark import model_metadata_registry
 from helm.benchmark.presentation.run_entry import RunEntry, read_run_entries
 from helm.common.cache_backend_config import MongoCacheBackendConfig, SqliteCacheBackendConfig
 from helm.common.general import ensure_directory_exists
-from helm.common.hierarchical_logger import hlog, htrack, htrack_block
+from helm.common.hierarchical_logger import hlog, htrack, htrack_block, setup_default_logging, hwarn
 from helm.common.authentication import Authentication
 from helm.common.object_spec import parse_object_spec, get_class_by_name
 from helm.proxy.services.remote_service import create_authentication, add_service_args
@@ -200,76 +200,9 @@ def validate_args(args):
 
 
 @htrack(None)
-def main():
-    parser = argparse.ArgumentParser()
-    add_service_args(parser)
-    parser.add_argument(
-        "-c",
-        "--conf-paths",
-        nargs="+",
-        help="Where to read RunSpecs to run from",
-        default=[],
-    )
-    parser.add_argument(
-        "--models-to-run",
-        nargs="+",
-        help="Only RunSpecs with these models specified. If no model is specified, runs with all models.",
-        default=None,
-    )
-    parser.add_argument(
-        "--groups-to-run",
-        nargs="+",
-        help="Only RunSpecs with these (scenario) groups specified. " "If no group is specified, runs with all groups.",
-        default=None,
-    )
-    parser.add_argument(
-        "--exit-on-error",
-        action="store_true",
-        help="Fail and exit immediately if a particular RunSpec fails.",
-    )
-    parser.add_argument(
-        "--skip-completed-runs",
-        action="store_true",
-        help="Skip RunSpecs that have completed i.e. output files exists.",
-    )
-    parser.add_argument(
-        "--priority",
-        type=int,
-        default=None,
-        help="Run RunSpecs with priority less than or equal to this number. "
-        "If a value for --priority is not specified, run on everything",
-    )
-    parser.add_argument(
-        "--run-specs",
-        nargs="*",
-        help="DEPRECATED: Use --run-entries instead. Will be removed in a future release. "
-        "Specifies run entries to run.",
-        default=[],
-    )
-    parser.add_argument("-r", "--run-entries", nargs="*", help="Specifies run entries to run", default=[])
-    parser.add_argument(
-        "--enable-huggingface-models",
-        nargs="+",
-        default=[],
-        help="Experimental: Enable using AutoModelForCausalLM models from Hugging Face Model Hub. "
-        "Format: namespace/model_name[@revision]",
-    )
-    parser.add_argument(
-        "--enable-local-huggingface-models",
-        nargs="+",
-        default=[],
-        help="Experimental: Enable using AutoModelForCausalLM models from a local path.",
-    )
-    parser.add_argument(
-        "--runner-class-name",
-        type=str,
-        default=None,
-        help="Full class name of the Runner class to use. If unset, uses the default Runner.",
-    )
-    add_run_args(parser)
-    args = parser.parse_args()
-    validate_args(args)
+def helm_run(args):
 
+    validate_args(args)
     register_builtin_configs_from_helm_package()
     register_configs_from_directory(args.local_path)
 
@@ -358,13 +291,91 @@ def main():
     )
 
     if args.run_specs:
-        hlog(
-            "WARNING: The --run-specs flag is deprecated and will be removed in a future release. "
-            "Use --run-entries instead."
+        hwarn(
+            "The --run-specs flag is deprecated and will be removed in a future release. " "Use --run-entries instead."
         )
 
     hlog("Done.")
 
 
+# Separate parsing from starting HELM so we can setup logging
+def main():
+    parser = argparse.ArgumentParser()
+    add_service_args(parser)
+    parser.add_argument(
+        "-c",
+        "--conf-paths",
+        nargs="+",
+        help="Where to read RunSpecs to run from",
+        default=[],
+    )
+    parser.add_argument(
+        "--models-to-run",
+        nargs="+",
+        help="Only RunSpecs with these models specified. If no model is specified, runs with all models.",
+        default=None,
+    )
+    parser.add_argument(
+        "--groups-to-run",
+        nargs="+",
+        help="Only RunSpecs with these (scenario) groups specified. " "If no group is specified, runs with all groups.",
+        default=None,
+    )
+    parser.add_argument(
+        "--exit-on-error",
+        action="store_true",
+        help="Fail and exit immediately if a particular RunSpec fails.",
+    )
+    parser.add_argument(
+        "--skip-completed-runs",
+        action="store_true",
+        help="Skip RunSpecs that have completed i.e. output files exists.",
+    )
+    parser.add_argument(
+        "--priority",
+        type=int,
+        default=None,
+        help="Run RunSpecs with priority less than or equal to this number. "
+        "If a value for --priority is not specified, run on everything",
+    )
+    parser.add_argument(
+        "--run-specs",
+        nargs="*",
+        help="DEPRECATED: Use --run-entries instead. Will be removed in a future release. "
+        "Specifies run entries to run.",
+        default=[],
+    )
+    parser.add_argument("-r", "--run-entries", nargs="*", help="Specifies run entries to run", default=[])
+    parser.add_argument(
+        "--enable-huggingface-models",
+        nargs="+",
+        default=[],
+        help="Experimental: Enable using AutoModelForCausalLM models from Hugging Face Model Hub. "
+        "Format: namespace/model_name[@revision]",
+    )
+    parser.add_argument(
+        "--enable-local-huggingface-models",
+        nargs="+",
+        default=[],
+        help="Experimental: Enable using AutoModelForCausalLM models from a local path.",
+    )
+    parser.add_argument(
+        "--runner-class-name",
+        type=str,
+        default=None,
+        help="Full class name of the Runner class to use. If unset, uses the default Runner.",
+    )
+    parser.add_argument(
+        "--log-config",
+        type=str,
+        default=None,
+        help="PATH to a YAML file to customize logging",
+    )
+    add_run_args(parser)
+    args = parser.parse_args()
+    setup_default_logging(args.log_config)
+    return helm_run(args)
+
+
 if __name__ == "__main__":
     main()

helm/benchmark/run_expander.py

@@ -21,7 +21,10 @@ from helm.benchmark.model_metadata_registry import (
     AUDIO_LANGUAGE_MODEL_TAG,
     INSTRUCTION_FOLLOWING_MODEL_TAG,
 )
-from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_GENERATION
+from helm.benchmark.adaptation.adapters.adapter_factory import (
+    ADAPT_GENERATION,
+    ADAPT_MULTIPLE_CHOICE_JOINT_CHAIN_OF_THOUGHT,
+)
 from helm.benchmark.model_deployment_registry import get_model_names_with_tokenizer
 from helm.benchmark.run_spec import RunSpec
 from helm.benchmark.adaptation.adapter_spec import ADAPT_MULTIPLE_CHOICE_JOINT, AdapterSpec, Substitution
@@ -537,6 +540,7 @@ class MaxTrainInstancesRunExpander(ReplaceValueRunExpander):
         "all": [0, 1, 2, 4, 8, 16],  # Cap at 16 due to limited context length
         "big_bench_few_shot_setting": [0, 1, 2, 3],  # Commonly used few-shot setting in BIG-bench
         "vhelm": [0, 1, 2, 4, 8],
+        "melt": [0, 1, 5],
     }
 
 
@@ -1476,6 +1480,8 @@ class OutputFormatInstructions(RunExpander):
                 instructions = "Answer with only a single letter."
             elif self.scenario == "mcqa":
                 instructions = "Answer with only a single letter."
+            elif self.scenario == "mcqa_no_period":
+                instructions = "Answer with only a single letter. Do not include a period in your answer."
             elif self.scenario == "mcqa_only_last_question":
                 instructions = "Answer only the last question with only a single letter."
             else:
@@ -1521,6 +1527,11 @@ class OutputFormatInstructions(RunExpander):
                 )
             else:
                 raise ValueError(f"Unknown scenario {self.scenario}")
+        elif run_spec.adapter_spec.method == ADAPT_MULTIPLE_CHOICE_JOINT_CHAIN_OF_THOUGHT:
+            if self.scenario == "mmlu_pro" or self.scenario == "gpqa":
+                instructions = 'In your response, replace "insert answer here" with the single uppercase letter corresponding to your answer.'  # noqa: E501
+            else:
+                raise ValueError(f"Unknown scenario {self.scenario}")
 
         if self.no_prefix:
             if instructions:

helm/benchmark/run_spec_factory.py

@@ -143,12 +143,13 @@ def construct_run_specs(spec: ObjectSpec) -> List[RunSpec]:
         ):
             run_spec = singleton(IncreaseMaxTokensRunExpander(value=1).expand(run_spec))
 
-        if model.name == "openai/o1-2024-12-17":
-            # From https://platform.openai.com/docs/guides/reasoning,
-            # "OpenAI recommends reserving at least 25,000 tokens for reasoning and outputs when you start
-            # experimenting with these models. As you become familiar with the number of reasoning tokens your
-            # prompts require, you can adjust this buffer accordingly."
-            run_spec = singleton(IncreaseMaxTokensRunExpander(value=25_000).expand(run_spec))
+        # TODO: find a better solution for this
+        # if model.name.startswith("openai/o"):
+        #     # From https://platform.openai.com/docs/guides/reasoning,
+        #     # "OpenAI recommends reserving at least 25,000 tokens for reasoning and outputs when you start
+        #     # experimenting with these models. As you become familiar with the number of reasoning tokens your
+        #     # prompts require, you can adjust this buffer accordingly."
+        #     run_spec = singleton(IncreaseMaxTokensRunExpander(value=25_000).expand(run_spec))
 
         # IDEFICS special handling
         if IDEFICS_MODEL_TAG in model.tags:

helm/benchmark/run_specs/arabic_run_specs.py

@@ -0,0 +1,73 @@
+"""Run specs for Arabic leaderboard
+
+EXPERIMENTAL: Run specs here may have future reverse incompatible changes."""
+
+from helm.benchmark.adaptation.adapter_spec import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec, get_generation_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("arabic_mmlu")
+def get_arabic_mmlu_spec() -> RunSpec:
+    """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.arabic_mmlu_scenario.ArabicMMLUScenario")
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="The following are multiple choice questions. Answer the last question. Respond only with only a single letter corresponding to your choice.",  # noqa: E501
+        input_noun="Question",
+        output_noun="Answer",
+    )
+
+    return RunSpec(
+        name="arabic_mmlu",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["arabic_mmlu"],
+    )
+
+
+@run_spec_function("alghafa")
+def get_alghafa_spec(subset: str) -> RunSpec:
+    """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.alghafa_scenario.AlGhafaScenario", args={"subset": subset}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="The following are multiple choice questions. Answer the last question. Respond only with only a single letter corresponding to your choice.",  # noqa: E501
+        input_noun="Question",
+        output_noun="Answer",
+    )
+
+    return RunSpec(
+        name=f"alghafa:subset={subset}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["alghafa", f"alghafa_{subset}"],
+    )
+
+
+@run_spec_function("aratrust")
+def get_aratrust_spec() -> RunSpec:
+    """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.aratrust_scenario.AraTrustScenario")
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions="The following are multiple choice questions. Answer the last question. Respond only with only a single letter corresponding to your choice.",  # noqa: E501
+        input_noun="Question",
+        output_noun="Answer",
+    )
+
+    return RunSpec(
+        name="aratrust",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["aratrust"],
+    )

helm/benchmark/run_specs/audio_run_specs.py

@@ -113,6 +113,18 @@ def _get_gpt4_critique_metric_specs(num_respondents: int, max_tokens: int) -> Li
     ]
 
 
+def _get_gpt4_refusal_metric_specs(num_respondents: int, max_tokens: int) -> List[MetricSpec]:
+    return [
+        MetricSpec(
+            class_name="helm.benchmark.metrics.gpt4_audio_refusal_metrics.GPT4AudioRefusalCritiqueMetric",
+            args={
+                "num_respondents": num_respondents,
+                "max_tokens": max_tokens,
+            },
+        )
+    ]
+
+
 ########################################################################################################################
 # RunSpecs
 
@@ -215,16 +227,20 @@ def get_mustard_audio_run_spec() -> RunSpec:
 
 
 @run_spec_function("voice_jailbreak_attacks")
-def get_voice_jailbreak_attacks_run_spec(subset: str) -> RunSpec:
+def get_voice_jailbreak_attacks_run_spec(subset: str, num_respondents: int = 1) -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.audio_language.voice_jailbreak_attacks_scenario."
         "VoiceJailbreakAttacksScenario",
         args={"subset": subset},
     )
-    adapter_spec = _get_generation_adapter_spec(max_tokens=1024)
-    metric_specs: List[MetricSpec] = get_generative_harms_metric_specs(
-        include_basic_metrics=True, include_generative_harms_metrics=True
+    adapter_spec = _get_generation_adapter_spec(
+        instructions="Listen to the audio and respond according to its instructions.",
+        max_tokens=1024,
     )
+    metric_specs: List[MetricSpec] = _get_gpt4_refusal_metric_specs(
+        num_respondents=num_respondents,
+        max_tokens=200,
+    ) + get_generative_harms_metric_specs(include_basic_metrics=True, include_generative_harms_metrics=True)
 
     run_spec_name: str = "voice_jailbreak_attacks"
     return RunSpec(
@@ -258,19 +274,20 @@ def get_covost2_run_spec(source_language: str, target_language: str) -> RunSpec:
 
 
 @run_spec_function("vocal_sound")
-def get_vocal_sound_run_spec() -> RunSpec:
+def get_vocal_sound_run_spec(sound: str) -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.audio_language.vocal_sound_scenario.VocalSoundScenario",
+        args={"sound": sound},
     )
     adapter_spec = _get_generation_adapter_spec(
         instructions="Listen to the audio and classify the speaker behavior. Choose only from these options:"
         '"Cough", "Laughter", "Sigh", "Sneeze", "Sniff", or "Throat clearing". Respond with just the behavior.',
         max_tokens=5,
     )
-    metric_specs = get_exact_match_metric_specs() + get_classification_metric_specs()
+    metric_specs = get_exact_match_metric_specs()
     run_spec_name: str = "vocal_sound"
     return RunSpec(
-        name=run_spec_name,
+        name=f"{run_spec_name}:sound={sound}",
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=metric_specs,
@@ -501,13 +518,20 @@ def get_air_bench_chat_run_spec(subject: str, num_respondents: int = 1) -> RunSp
         )
         + _get_open_ended_generation_metric_specs()
     )
+
     run_spec_name: str = "air_bench_chat"
+    group_name: str = run_spec_name
+    if subject in ["mix", "speech"]:
+        group_name += "_reasoning"
+    elif subject in ["sound", "music"]:
+        group_name += "_knowledge"
+
     return RunSpec(
         name=f"{run_spec_name}:subject={subject}",
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=metric_specs,
-        groups=[run_spec_name],
+        groups=[group_name],
     )
 
 
@@ -611,3 +635,23 @@ def get_parade_run_spec(voice: str, subset: str) -> RunSpec:
         metric_specs=metric_specs,
         groups=[run_spec_name],
     )
+
+
+@run_spec_function("corebench")
+def get_corebench_run_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.audio_language.corebench_scenario.COREBenchScenario",
+    )
+    adapter_spec = _get_generation_adapter_spec(
+        instructions="",
+        max_tokens=10,
+    )
+    metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
+    run_spec_name: str = "corebench"
+    return RunSpec(
+        name=f"{run_spec_name}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )

helm/benchmark/run_specs/bluex_run_specs.py

@@ -0,0 +1,40 @@
+from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("bluex")
+def get_bluex_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bluex_scenario.BLUEX_Scenario", args={})
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="""
+        Escolha a alternativa correta para as questões de vestibulares (responda apenas com a letra).
+        Exemplo de Pergunta com a resposta:
+        Em um romance narrado em primeira pessoa, o narrador participa dos acontecimentos da trama,
+        relatando suas próprias experiências e sentimentos. Qual alternativa apresenta essa característica?
+
+        (A) Narrador onisciente que conhece os pensamentos de todas as personagens.
+        (B) Narrador que descreve os fatos de forma imparcial, sem envolvimento emocional.
+        (C) Narrador-personagem que vivencia e relata os eventos da história.
+        (D) Narrador observador que apenas registra as ações visíveis.
+        (E) Narrador em segunda pessoa que se dirige constantemente ao leitor.
+
+        Resposta correta: C
+
+        A partir disso, responda:
+        """,
+        input_noun="Pergunta",
+        output_noun="Resposta",
+    )
+
+    return RunSpec(
+        name="bluex",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["bluex"],
+    )

helm/benchmark/run_specs/classic_run_specs.py

@@ -35,7 +35,6 @@ from helm.benchmark.metrics.common_metric_specs import (
     get_f1_metric_specs,
     get_generative_harms_metric_specs,
     get_language_modeling_metric_specs,
-    get_numeracy_metric_specs,
     get_open_ended_generation_metric_specs,
     get_summarization_metric_specs,
     get_basic_generation_metric_specs,
@@ -381,58 +380,6 @@ def get_raft_spec(subset: str) -> RunSpec:
     )
 
 
-@run_spec_function("numeracy")
-def get_numeracy_spec(
-    relation_type: str = "linear", mode: str = "function", seed: str = "0", run_solver: str = "False"
-) -> RunSpec:
-    from helm.benchmark.scenarios.numeracy_scenario import get_numeracy_adapter_spec, RELTYPE_INFO
-
-    run_solver_bool: bool = True if run_solver.lower() == "true" else False
-    del run_solver
-    random_seed = int(seed)
-    scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.numeracy_scenario.NumeracyScenario",
-        args={"seed": random_seed, "relation_type": relation_type, "mode": mode},
-    )
-
-    if mode in ["example", "standard"]:
-        # Test a model's ability to impute datapoints for a given (example or randomly sampled) relation.
-        adapter_args: Dict[str, Any] = {
-            "max_train_instances": 100,
-            "max_eval_instances": 100,
-            "dim": RELTYPE_INFO[relation_type].num_variables + 1,
-        }
-    elif mode == "function":
-        # Test a model's ability to impute datapoints for randomly sampled relations
-        # (resampled for each evaluation point).
-        adapter_args = {
-            "instructions": "",
-            "max_train_instances": 0,  # Turn off general version of `function` mode because it doesn't cleanly
-            # capture a higher-order version of this task / is a little convoluted
-            # for models, currently.
-            # (In the general version, the model sees other relations of the same class,
-            # and needs to impute a datapoint for the last one. Presumably, inferring
-            # the class - eg. the degree of the relation - would help.)
-            "max_eval_instances": 1000,
-            "dim": RELTYPE_INFO[relation_type].num_variables + 1,
-            "instance_prefix": "\n\n",
-        }
-    else:
-        raise ValueError(f"Invalid mode: {mode}")
-
-    adapter_spec = get_numeracy_adapter_spec(**adapter_args)  # Construct the AdapterSpec using a helper function.
-    # `get_numeracy_adapter_spec` is defined in numeracy_scenario.py
-    # because it is used within the scenario to construct the instances themselves.
-
-    return RunSpec(
-        name=f"numeracy:relation_type={relation_type},mode={mode}",
-        scenario_spec=scenario_spec,
-        adapter_spec=adapter_spec,
-        metric_specs=get_numeracy_metric_specs(run_solver_bool),
-        groups=["numeracy"],
-    )
-
-
 @run_spec_function("boolq")
 def get_boolq_spec(only_contrast=False) -> RunSpec:
     scenario_spec = ScenarioSpec(