crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/run_spec.py

@@ -0,0 +1,93 @@
+from dataclasses import dataclass, field
+import importlib
+import os
+import pkgutil
+from typing import Callable, Dict, Iterable, List, Optional, TypeVar
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.augmentations.data_augmenter import DataAugmenterSpec
+from helm.benchmark.metrics.metric import MetricSpec
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+from helm.benchmark.annotation.annotator import AnnotatorSpec
+
+
+@dataclass(frozen=True)
+class RunSpec:
+    """
+    Specifies how to do a single run, which gets a scenario, adapts it, and
+    computes a list of stats based on the defined metrics.
+    """
+
+    name: str
+    """Unique identifier of the RunSpec"""
+
+    scenario_spec: ScenarioSpec
+    """Which scenario"""
+
+    adapter_spec: AdapterSpec
+    """Specifies how to adapt an instance into a set of requests"""
+
+    metric_specs: List[MetricSpec]
+    """What to evaluate on"""
+
+    data_augmenter_spec: DataAugmenterSpec = DataAugmenterSpec()
+    """Data augmenter. The default `DataAugmenterSpec` does nothing."""
+
+    groups: List[str] = field(default_factory=list)
+    """Groups that this run spec belongs to (for aggregation)"""
+
+    annotators: Optional[List[AnnotatorSpec]] = None
+    """Annotators to use for this run spec"""
+
+    def __post_init__(self):
+        """
+        `self.name` is used as the name of the output folder for the `RunSpec`.
+        Clean up `self.name` by replacing any "/"'s with "_".
+        """
+        # TODO: Don't mutate name! clean this up before passing it into the constructor here
+        object.__setattr__(self, "name", self.name.replace(os.path.sep, "_"))
+
+
+RunSpecFunction = Callable[..., RunSpec]
+
+
+_REGISTERED_RUN_SPEC_FUNCTIONS: Dict[str, RunSpecFunction] = {}
+"""Dict of run spec function names to run spec functions."""
+
+
+F = TypeVar("F", bound=RunSpecFunction)
+
+
+def run_spec_function(name: str) -> Callable[[F], F]:
+    """Register the run spec function under the given name."""
+
+    def wrap(func: F) -> F:
+        if name in _REGISTERED_RUN_SPEC_FUNCTIONS:
+            raise ValueError(f"A run spec function with name {name} already exists")
+        _REGISTERED_RUN_SPEC_FUNCTIONS[name] = func
+        return func
+
+    return wrap
+
+
+# Copied from https://docs.python.org/3/library/pkgutil.html#pkgutil.iter_modules
+def _iter_namespace(ns_pkg) -> Iterable[pkgutil.ModuleInfo]:
+    # Specifying the second argument (prefix) to iter_modules makes the
+    # returned name an absolute name instead of a relative one. This allows
+    # import_module to work without having to do additional modification to
+    # the name.
+    return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".")
+
+
+def discover_run_spec_functions() -> None:
+    """Discover and register all run spec functions under helm.benchmark.run_specs"""
+    import helm.benchmark.run_specs  # noqa
+
+    for finder, name, ispkg in _iter_namespace(helm.benchmark.run_specs):
+        importlib.import_module(name)
+
+
+def get_run_spec_function(name: str) -> Optional[RunSpecFunction]:
+    """Return the run spec function registered under the given name."""
+    discover_run_spec_functions()
+    return _REGISTERED_RUN_SPEC_FUNCTIONS.get(name)

helm/benchmark/run_spec_factory.py

@@ -0,0 +1,180 @@
+import dataclasses
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import (
+    ADAPT_GENERATION,
+    ADAPT_MULTIPLE_CHOICE_JOINT,
+)
+from helm.benchmark.model_deployment_registry import (
+    ModelDeployment,
+    get_default_model_deployment_for_model,
+    get_model_deployment,
+)
+from helm.benchmark.model_metadata_registry import (
+    ANTHROPIC_CLAUDE_1_MODEL_TAG,
+    ANTHROPIC_CLAUDE_2_MODEL_TAG,
+    ANTHROPIC_CLAUDE_3_MODEL_TAG,
+    BUGGY_TEMP_0_TAG,
+    CHATML_MODEL_TAG,
+    GOOGLE_GEMINI_PRO_VISION_V1_TAG,
+    IDEFICS_INSTRUCT_MODEL_TAG,
+    LLAVA_MODEL_TAG,
+    OPEN_FLAMINGO_MODEL_TAG,
+    NLG_PREFIX_TAG,
+    NO_NEWLINES_TAG,
+    VISION_LANGUAGE_MODEL_TAG,
+    IDEFICS_MODEL_TAG,
+    ModelMetadata,
+    get_model_metadata,
+)
+from helm.benchmark.run_expander import (
+    RUN_EXPANDERS,
+    AnthropicClaude2RunExpander,
+    AnthropicClaude3RunExpander,
+    ChatMLRunExpander,
+    GlobalPrefixRunExpander,
+    IDEFICSInstructRunExpander,
+    IncreaseTemperatureRunExpander,
+    IncreaseMaxTokensRunExpander,
+    LlavaRunExpander,
+    ModelRunExpander,
+    OpenFlamingoRunExpander,
+    StopRunExpander,
+)
+from helm.benchmark.run_spec import RunSpec, get_run_spec_function
+from helm.common.general import singleton
+from helm.common.object_spec import ObjectSpec
+
+
+def construct_run_specs(spec: ObjectSpec) -> List[RunSpec]:
+    """
+    Takes a specification (name, args) and returns a list of `RunSpec`s.
+    """
+    # Note that we are abusing `spec` a bit because the name is not actually a class name.
+    name = spec.class_name
+    args = spec.args
+
+    run_spec_function = get_run_spec_function(name)
+    if run_spec_function is None:
+        raise ValueError(f"Unknown run spec name: {name}")
+
+    # Peel off the run expanders (e.g., model)
+    expanders = [RUN_EXPANDERS[key](value) for key, value in args.items() if key in RUN_EXPANDERS]  # type: ignore
+    args = dict((key, value) for key, value in args.items() if key not in RUN_EXPANDERS)
+
+    # If no model run expander was specified, add the model=all run expander
+    if not any([expander for expander in expanders if isinstance(expander, ModelRunExpander)]):
+        expanders.append(ModelRunExpander("all"))
+
+    run_specs: List[RunSpec] = [run_spec_function(**args)]
+
+    # Apply expanders
+    for expander in expanders:
+        run_specs = [
+            child_run_spec for parent_run_spec in run_specs for child_run_spec in expander.expand(parent_run_spec)
+        ]
+
+    def alter_run_spec(run_spec: RunSpec) -> RunSpec:
+        if not run_spec.adapter_spec.model and not run_spec.adapter_spec.model_deployment:
+            raise ValueError("At least one of model_deployment and model must be specified")
+        elif not run_spec.adapter_spec.model and run_spec.adapter_spec.model_deployment:
+            # Infer model from model deployment
+            default_model_name = get_model_deployment(run_spec.adapter_spec.model_deployment).model_name
+            if not default_model_name:
+                default_model_name = run_spec.adapter_spec.model_deployment
+            run_spec = dataclasses.replace(
+                run_spec,
+                adapter_spec=dataclasses.replace(run_spec.adapter_spec, model=default_model_name),
+            )
+        elif run_spec.adapter_spec.model and not run_spec.adapter_spec.model_deployment:
+            # Infer model deployment from model
+            default_model_deployment = get_default_model_deployment_for_model(run_spec.adapter_spec.model)
+            if not default_model_deployment:
+                raise ValueError(
+                    f"Unknown model or no default model deployment found for model {run_spec.adapter_spec.model}"
+                )
+            run_spec = dataclasses.replace(
+                run_spec,
+                adapter_spec=dataclasses.replace(run_spec.adapter_spec, model_deployment=default_model_deployment),
+            )
+
+        # Both model and model_deployment should now be filled
+        assert run_spec.adapter_spec.model_deployment
+        assert run_spec.adapter_spec.model
+
+        model: ModelMetadata = get_model_metadata(run_spec.adapter_spec.model)
+        deployment: ModelDeployment = get_model_deployment(run_spec.adapter_spec.model_deployment)
+        if run_spec.adapter_spec.model != deployment.model_name:
+            raise ValueError(
+                f"Invalid RunSpec: selected model deployment '{run_spec.adapter_spec.model_deployment}'"
+                f"for model '{run_spec.adapter_spec.model}' but the model deployment is "
+                f"for a different model '{deployment.model_name}'"
+            )
+        # For models that strip newlines, when we're generating, we need to set
+        # the delimiter to be '###' so we stop properly.
+        if NO_NEWLINES_TAG in model.tags and run_spec.adapter_spec.method in (
+            ADAPT_GENERATION,
+            ADAPT_MULTIPLE_CHOICE_JOINT,
+        ):
+            stop_expander = StopRunExpander(value="hash")
+            run_spec = singleton(stop_expander.expand(run_spec))
+
+        if NLG_PREFIX_TAG in model.tags:
+            global_prefix_expander = GlobalPrefixRunExpander(value="nlg")
+            run_spec = singleton(global_prefix_expander.expand(run_spec))
+
+        if CHATML_MODEL_TAG in model.tags:
+            chatml_expander = ChatMLRunExpander()
+            run_spec = singleton(chatml_expander.expand(run_spec))
+
+        # Anthropic Claude 1 and 2 prompts
+        if ANTHROPIC_CLAUDE_1_MODEL_TAG in model.tags or ANTHROPIC_CLAUDE_2_MODEL_TAG in model.tags:
+            run_spec = singleton(AnthropicClaude2RunExpander().expand(run_spec))
+
+        # Anthropic Claude 3
+        if ANTHROPIC_CLAUDE_3_MODEL_TAG in model.tags:
+            run_spec = singleton(AnthropicClaude3RunExpander().expand(run_spec))
+
+        # Google Gemini Vision v1.0 returns an empty completion or throws an error if max_tokens is 1
+        if (
+            VISION_LANGUAGE_MODEL_TAG in model.tags
+            and GOOGLE_GEMINI_PRO_VISION_V1_TAG in model.tags
+            and run_spec.adapter_spec.max_tokens == 1
+        ):
+            run_spec = singleton(IncreaseMaxTokensRunExpander(value=1).expand(run_spec))
+
+        # TODO: find a better solution for this
+        # if model.name.startswith("openai/o"):
+        #     # From https://platform.openai.com/docs/guides/reasoning,
+        #     # "OpenAI recommends reserving at least 25,000 tokens for reasoning and outputs when you start
+        #     # experimenting with these models. As you become familiar with the number of reasoning tokens your
+        #     # prompts require, you can adjust this buffer accordingly."
+        #     run_spec = singleton(IncreaseMaxTokensRunExpander(value=25_000).expand(run_spec))
+
+        # IDEFICS special handling
+        if IDEFICS_MODEL_TAG in model.tags:
+            if IDEFICS_INSTRUCT_MODEL_TAG in model.tags:
+                run_spec = singleton(IDEFICSInstructRunExpander().expand(run_spec))
+
+        # Llava
+        if LLAVA_MODEL_TAG in model.tags:
+            run_spec = singleton(LlavaRunExpander().expand(run_spec))
+
+        # OpenFlamingo
+        if OPEN_FLAMINGO_MODEL_TAG in model.tags:
+            run_spec = singleton(OpenFlamingoRunExpander().expand(run_spec))
+
+        # For multiple choice
+        if BUGGY_TEMP_0_TAG in model.tags and run_spec.adapter_spec.temperature == 0:
+            increase_temperature_expander = IncreaseTemperatureRunExpander(value=1e-4)
+            run_spec = singleton(increase_temperature_expander.expand(run_spec))
+
+        # MedLM-Large
+        if run_spec.adapter_spec.model == "google/medlm-large":
+            run_spec = singleton(StopRunExpander("none").expand(run_spec))
+
+        return run_spec
+
+    run_specs = [alter_run_spec(run_spec) for run_spec in run_specs]
+
+    return run_specs

helm/benchmark/run_specs/air_bench_run_specs.py

@@ -0,0 +1,58 @@
+from typing import Dict, Optional
+
+from helm.benchmark.adaptation.adapter_spec import ADAPT_GENERATION, AdapterSpec
+from helm.benchmark.annotation.annotator import AnnotatorSpec
+from helm.benchmark.metrics.metric import MetricSpec
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("air_bench_2024")
+def get_air_bench_2024_spec(
+    annotator_model: Optional[str] = None, annotator_model_deployment: Optional[str] = None
+) -> RunSpec:
+    run_spec_name = "air_bench_2024"
+    adapter_spec = AdapterSpec(
+        method=ADAPT_GENERATION,
+        global_prefix="",
+        global_suffix="",
+        instructions="",
+        input_prefix="",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        instance_prefix="",
+        max_train_instances=0,
+        num_outputs=1,
+        max_tokens=512,
+        temperature=0.0,
+        stop_sequences=[],
+    )
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.air_bench_scenario.AIRBench2024Scenario")
+    annotator_args: Dict[str, str] = {}
+    if annotator_model:
+        annotator_args["model"] = annotator_model
+        annotator_args["model_deployment"] = annotator_model_deployment or annotator_model
+        run_spec_name = (
+            "air_bench_2024:"
+            f"annotator_model={annotator_args['model']},"
+            f"annotator_model_deployment={annotator_args['model_deployment']}"
+        )
+    annotator_specs = [
+        AnnotatorSpec(
+            class_name="helm.benchmark.annotation.air_bench_annotator.AIRBench2024Annotator", args=annotator_args
+        )
+    ]
+    metric_specs = [
+        MetricSpec(class_name="helm.benchmark.metrics.air_bench_metrics.AIRBench2024ScoreMetric"),
+        MetricSpec(class_name="helm.benchmark.metrics.air_bench_metrics.AIRBench2024BasicGenerationMetric"),
+        MetricSpec(class_name="helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric"),
+    ]
+    return RunSpec(
+        name=run_spec_name,
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        annotators=annotator_specs,
+        groups=["air_bench_2024"],
+    )

helm/benchmark/run_specs/arabic_run_specs.py

@@ -0,0 +1,197 @@
+"""Run specs for Arabic leaderboard
+
+EXPERIMENTAL: Run specs here may have future reverse incompatible changes."""
+
+from helm.benchmark.adaptation.adapter_spec import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec, get_generation_adapter_spec
+from helm.benchmark.annotation.annotator import AnnotatorSpec
+from helm.benchmark.metrics.common_metric_specs import get_basic_metric_specs, get_exact_match_metric_specs
+from helm.benchmark.metrics.metric import MetricSpec
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+_ARABIC_REFERENCE_PREFIX_CHARACTERS = ["أ", "ب", "ج", "د", "هـ"]
+_ARABIC_OUTPUT_MAPPING_PATTERN = "(أ|ب|ج|د|هـ)"
+
+
+@run_spec_function("arabic_mmlu")
+def get_arabic_mmlu_spec(subset: str) -> RunSpec:
+    """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
+
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.arabic_mmlu_scenario.ArabicMMLUScenario", args={"subset": subset}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+        output_mapping_pattern=_ARABIC_OUTPUT_MAPPING_PATTERN,
+    )
+
+    return RunSpec(
+        name=f"arabic_mmlu:subset={subset}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["arabic_mmlu"],
+    )
+
+
+@run_spec_function("alghafa")
+def get_alghafa_spec(subset: str) -> RunSpec:
+    """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.alghafa_scenario.AlGhafaScenario", args={"subset": subset}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+        output_mapping_pattern=_ARABIC_OUTPUT_MAPPING_PATTERN,
+    )
+
+    return RunSpec(
+        name=f"alghafa:subset={subset}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["alghafa", f"alghafa_{subset}"],
+    )
+
+
+@run_spec_function("aratrust")
+def get_aratrust_spec(category: str) -> RunSpec:
+    """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.aratrust_scenario.AraTrustScenario",
+        args={"category": category},
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة: أ، ب أو ج",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+    )
+
+    return RunSpec(
+        name=f"aratrust:category={category}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["aratrust"],
+    )
+
+
+@run_spec_function("alrage")
+def get_alrage_spec() -> RunSpec:
+    """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.alrage_scenario.ALRAGEScenario")
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions="بناءً على السياقات المقترحة التالية، اجب عن السؤال التالي",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+    )
+
+    annotator_specs = [AnnotatorSpec(class_name="helm.benchmark.annotation.alrage_annotator.ALRAGEAnnotator")]
+
+    metric_specs = [
+        MetricSpec(class_name="helm.benchmark.metrics.alrage_metric.ALRAGEMetric")
+    ] + get_basic_metric_specs([])
+
+    return RunSpec(
+        name="alrage",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        annotators=annotator_specs,
+        metric_specs=metric_specs,
+        groups=["alrage"],
+    )
+
+
+@run_spec_function("madinah_qa")
+def get_madinah_qa_spec(subset: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.madinah_qa_scenario.MadinahQAScenario", args={"subset": subset}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+        output_mapping_pattern=_ARABIC_OUTPUT_MAPPING_PATTERN,
+    )
+
+    return RunSpec(
+        name=f"madinah_qa:subset={subset}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["madinah_qa"],
+    )
+
+
+@run_spec_function("mbzuai_human_translated_arabic_mmlu")
+def get_arabic_mmmlu_spec(subject: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.mbzuai_human_translated_arabic_mmlu.MBZUAIHumanTranslatedArabicMMLUScenario",
+        args={"subject": subject},
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+        output_mapping_pattern=_ARABIC_OUTPUT_MAPPING_PATTERN,
+    )
+
+    return RunSpec(
+        name=f"mbzuai_human_translated_arabic_mmlu:subject={subject}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["mbzuai_human_translated_arabic_mmlu"],
+    )
+
+
+@run_spec_function("arabic_exams")
+def get_arabic_exams_spec(subject: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.arabic_exams_scenario.ArabicEXAMSScenario",
+        args={"subject": subject},
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+        output_mapping_pattern=_ARABIC_OUTPUT_MAPPING_PATTERN,
+    )
+
+    return RunSpec(
+        name=f"arabic_exams:subject={subject}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["arabic_exams"],
+    )