crfm-helm 0.5.2__py3-none-any.whl → 0.5.3__py3-none-any.whl

helm/benchmark/metrics/annotation_metrics.py

@@ -0,0 +1,108 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class AnnotationLabelMetric(Metric):
+    """Binary metric for labels produced by annotators.
+
+    Expects the annotation with the given annotator name and key to be a string label.
+
+    For each possible label in the list of possible labels, produces a
+    corresponding stat with a value of 1 or 0 indicating if the actual label
+    in the annoation."""
+
+    def __init__(self, annotator_name: str, key: str, labels: List[str]):
+        super().__init__()
+        self.annotator_name = annotator_name
+        self.key = key
+        self.labels = labels
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotation_label = request_state.annotations[self.annotator_name][self.key]
+        if annotation_label not in self.labels:
+            raise ValueError(
+                f"Unrecognized annotation label '{annotation_label}' "
+                f"(known labels: {self.labels}) "
+                f"in annotation {request_state.annotations[self.annotator_name]} "
+                f"for instance id {request_state.instance.id}"
+            )
+        stats: List[Stat] = []
+        for label in self.labels:
+            stats.append(
+                Stat(MetricName(f"annotation_{self.annotator_name}_{self.key}_{label}")).add(
+                    1 if label == annotation_label else 0
+                )
+            )
+        return stats
+
+
+class AnnotationNumericMetric(Metric):
+    """Numeric metric for numbers produced by annotators.
+
+    Expects the annotation with the given annotator name and key to be a number."""
+
+    def __init__(self, annotator_name: str, key: str):
+        super().__init__()
+        self.annotator_name = annotator_name
+        self.key = key
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        score = request_state.annotations[self.annotator_name][self.key]
+        return [Stat(MetricName(f"annotation_{self.annotator_name}_{self.key}")).add(score)]
+
+
+class AnnotationLikertScaleMetric(Metric):
+    """Numeric metric for labels produced by annotators.
+
+    Expects the annotation with the given annotator name and key to be a string label.
+
+    For each possible label in the list of possible labels, produces a
+    corresponding stat with a value of 1 or 0 indicating if the actual label
+    in the annoation."""
+
+    def __init__(self, annotator_name: str, key: str, min_score: int, max_score: int):
+        super().__init__()
+        self.annotator_name = annotator_name
+        self.key = key
+        self.min_score = min_score
+        self.max_score = max_score
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        likert_score = request_state.annotations[self.annotator_name][self.key]
+        if likert_score < self.min_score or likert_score > self.max_score:
+            raise ValueError(
+                f"Likert score {likert_score} "
+                f"out of bounds {self.min_score} to {self.max_score} "
+                f"under key {self.key} and annotator {self.annotator_name} "
+                f"in annotation {request_state.annotations[self.annotator_name]} "
+                f"for instance id {request_state.instance.id}"
+            )
+        normalized_score = (likert_score - self.min_score) / (self.max_score - self.min_score)
+        return [Stat(MetricName(f"annotation_{self.annotator_name}_{self.key}")).add(normalized_score)]

helm/benchmark/metrics/bhasa_metrics.py

@@ -0,0 +1,188 @@
+import re
+import string
+from typing import Callable, Dict, List
+from collections import Counter
+
+from pythainlp.tokenize import word_tokenize
+from sacrebleu.metrics import CHRF
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class BhasaMachineTranslationMetric(Metric):
+    """Machine Translation Metrics
+
+    This class computes the following standard machine translation metrics
+
+    1. chr_f_plus_plus (ChrF++)
+
+    @inproceedings{popovic-2015-chrf,
+        title = "chr{F}: character n-gram {F}-score for automatic {MT} evaluation",
+        author = "Popovi{\'c}, Maja",
+        editor = "Bojar, Ond{\v{r}}ej  and
+        Chatterjee, Rajan  and
+        Federmann, Christian  and
+        Haddow, Barry  and
+        Hokamp, Chris  and
+        Huck, Matthias  and
+        Logacheva, Varvara  and
+        Pecina, Pavel",
+        booktitle = "Proceedings of the Tenth Workshop on Statistical Machine Translation",
+        month = sep,
+        year = "2015",
+        address = "Lisbon, Portugal",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/W15-3049",
+        doi = "10.18653/v1/W15-3049",
+        pages = "392--395",
+        github = "https://github.com/mjpost/sacrebleu",
+    }
+    """
+
+    def __init__(self):
+        self.chrf_scorer = CHRF(word_order=2)
+
+    def chr_f_plus_plus(self, refs: List[str], pred: str) -> Dict[str, float]:
+        metrics: Dict[str, float] = {}
+        metrics["chr_f_plus_plus"] = self.chrf_scorer.sentence_score(pred, refs).score
+        return metrics
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        refs: List[str] = [ref.output.text for ref in request_state.instance.references]
+
+        assert request_state.result is not None
+        pred: str = request_state.result.completions[0].text.strip()
+
+        result: List[Stat] = []
+
+        # Compute ChrF++ metrics
+        result.extend(
+            [Stat(MetricName(name)).add(float(val)) for name, val in self.chr_f_plus_plus(refs, pred).items()]
+        )
+
+        return result
+
+
+class BhasaQAMetric(Metric):
+    """Bhasa QA Metrics
+
+    This class computes the following standard SQuAD v1.1 metrics
+
+    1. squad_exact_match_score (SQuAD exact match score)
+    2. squad_f1_score (SQuAD macro-averaged F1 score)
+
+    @inproceedings{rajpurkar-etal-2016-squad,
+        title = "{SQ}u{AD}: 100,000+ Questions for Machine Comprehension of Text",
+        author = "Rajpurkar, Pranav  and
+            Zhang, Jian  and
+            Lopyrev, Konstantin  and
+            Liang, Percy",
+        editor = "Su, Jian  and
+            Duh, Kevin  and
+            Carreras, Xavier",
+        booktitle = "Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing",
+        month = nov,
+        year = "2016",
+        address = "Austin, Texas",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/D16-1264",
+        doi = "10.18653/v1/D16-1264",
+        pages = "2383--2392",
+    }
+    """
+
+    def __init__(self, language: str = "en"):
+        self.language: str = language
+        self.metrics: Dict[str, Callable[[str, str], float]] = {
+            "squad_exact_match_score": self.squad_exact_match_score,
+            "squad_f1_score": self.squad_f1_score,
+        }
+
+    def normalize_answer(self, text: str) -> List[str]:
+        """
+        For Thai, this will split the text using PyThaiNLP's tokenizer.
+        For all other languages, this will:
+        - Lower text
+        - Remove punctuation
+        - Remove extra whitespace
+
+        If the language is English, it will
+        - Remove articles "a", "an", and "the"
+
+        Modifies code from [SQuAD v1.1](https://github.com/allenai/bi-att-flow/blob/master/squad/evaluate-v1.1.py).
+        """
+
+        def remove_articles(text: str) -> str:
+            return re.sub(r"\b(a|an|the)\b", " ", text)
+
+        # This function is implemented to match SQuAD v1.1 behavior
+        def white_space_fix(text: str) -> str:
+            return " ".join(text.split())
+
+        def remove_punc(text: str) -> str:
+            exclude = set(string.punctuation)
+            return "".join(ch for ch in text if ch not in exclude)
+
+        def lower(text: str) -> str:
+            return text.lower()
+
+        normalized_text = remove_punc(lower(text))
+        if self.language == "th":
+            return word_tokenize(normalized_text, engine="newmm")
+        elif self.language == "en":
+            return white_space_fix(remove_articles(normalized_text)).split()
+        else:
+            return white_space_fix(normalized_text).split()
+
+    def squad_f1_score(self, gold: str, pred: str) -> float:
+        prediction_tokens = self.normalize_answer(pred)
+        ground_truth_tokens = self.normalize_answer(gold)
+        common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
+        num_same = sum(common.values())
+        if num_same == 0:
+            return 0
+        precision = 1.0 * num_same / len(prediction_tokens)
+        recall = 1.0 * num_same / len(ground_truth_tokens)
+        f1 = (2 * precision * recall) / (precision + recall)
+        return f1
+
+    def squad_exact_match_score(self, gold: str, pred: str) -> float:
+        return self.normalize_answer(pred) == self.normalize_answer(gold)
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+
+        stats: List[Stat] = []
+        if len(request_state.instance.references) > 0:
+            golds = [reference for reference in request_state.instance.references if reference.is_correct]
+            assert len(golds) > 0
+
+            assert request_state.result is not None
+            sorted_completions = sorted(request_state.result.completions, key=lambda x: -x.logprob)
+            preds = [completion.text.strip() for completion in sorted_completions]
+
+            for name, metric in self.metrics.items():
+                score_1 = max(metric(gold.output.text.strip(), preds[0]) for gold in golds)
+                metrics = [Stat(MetricName(name)).add(score_1)]
+                if adapter_spec.num_outputs != 1:
+                    score_k = max(metric(gold.output.text.strip(), pred) for gold in golds for pred in preds)
+                    metrics.append(Stat(MetricName(f"{name}@{adapter_spec.num_outputs}")).add(score_k))
+                stats.extend(metrics)
+
+        return stats

helm/benchmark/metrics/bhasa_metrics_specs.py

@@ -0,0 +1,10 @@
+from typing import Any, Dict, List
+from helm.benchmark.metrics.metric import MetricSpec
+
+
+def get_bhasa_machine_translation_metric_specs() -> List[MetricSpec]:
+    return [MetricSpec(class_name="helm.benchmark.metrics.bhasa_metrics.BhasaMachineTranslationMetric")]
+
+
+def get_bhasa_qa_metric_specs(args: Dict[str, Any]) -> List[MetricSpec]:
+    return [MetricSpec(class_name="helm.benchmark.metrics.bhasa_metrics.BhasaQAMetric", args=args)]

helm/benchmark/metrics/code_metrics_helper.py

@@ -27,14 +27,24 @@ import signal
 import sys
 import tempfile
 from typing import List, Union, Dict, Optional
+from types import ModuleType
 from unittest.mock import patch, mock_open
 
 import numpy as np
-from pyext import RuntimeModule
 
 from helm.common.hierarchical_logger import hlog
 
 
+class RuntimeModule(ModuleType):
+    """crfm-helm's replacement for pyext.RuntimeModule, since pyext is not supported by Python >=3.11"""
+
+    @staticmethod
+    def from_string(module_name: str, module_doc: str, module_contents: str) -> "RuntimeModule":
+        module = RuntimeModule(module_name, module_doc)
+        exec(module_contents, module.__dict__)
+        return module
+
+
 # === APPS evaluation below ===
 class CodeType(Enum):
     call_based = 0

helm/benchmark/metrics/safety_metrics.py

@@ -0,0 +1,57 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.basic_metrics import compute_request_state_metrics
+from helm.benchmark.metrics.efficiency_metrics import EfficiencyMetric
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class SafetyBasicGenerationMetric(Metric):
+    """Replacement for BasicGenerationMetric for HELM Safety.
+    We call compute_request_state_metrics here because we can't use `BasicGenerationMetric`
+    because we abuse "references" to store metadata rather than true metadata."""
+
+    def __init__(self):
+        super().__init__()
+        self.efficiency_metric = EfficiencyMetric()
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        return compute_request_state_metrics(self.efficiency_metric, adapter_spec, request_state, metric_service)
+
+
+class SafetyScoreMetric(Metric):
+    """Score metrics for HELM Safety."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        safety_scenarios = ["harm_bench", "anthropic_red_team", "simple_safety_tests", "xstest"]
+        curr_scenario = None
+        for annotation_key in request_state.annotations:
+            if annotation_key in safety_scenarios:
+                curr_scenario = annotation_key
+        if curr_scenario is None:
+            raise ValueError("No safety scenario found in annotations")
+        gpt_score = request_state.annotations[curr_scenario]["gpt_score"]
+        llama_score = request_state.annotations[curr_scenario]["llama_score"]
+        score = (gpt_score + llama_score) / 2
+        return [
+            Stat(MetricName("safety_score")).add(score),
+            Stat(MetricName("safety_gpt_score")).add(gpt_score),
+            Stat(MetricName("safety_llama_score")).add(llama_score),
+        ]

helm/benchmark/metrics/summac/model_summac.py

@@ -179,9 +179,9 @@ class SummaCImager:
                     model_outputs = self.model(**batch_tokens)
 
                 batch_probs = torch.nn.functional.softmax(model_outputs["logits"], dim=-1)
-                batch_evids = batch_probs[:, self.entailment_idx].tolist()
-                batch_conts = batch_probs[:, self.contradiction_idx].tolist()
-                batch_neuts = batch_probs[:, self.neutral_idx].tolist()
+                batch_evids = batch_probs[:, self.entailment_idx].tolist()  # type: ignore
+                batch_conts = batch_probs[:, self.contradiction_idx].tolist()  # type: ignore
+                batch_neuts = batch_probs[:, self.neutral_idx].tolist()  # type: ignore
 
             for b, evid, cont, neut in zip(batch, batch_evids, batch_conts, batch_neuts):
                 image[0, b["doc_i"], b["gen_i"]] = evid

helm/benchmark/metrics/tokens/test_ai21_token_cost_estimator.py

@@ -10,8 +10,8 @@ class TestAI21TokenCostEstimator:
 
     def test_estimate_tokens(self):
         request = Request(
-            model="openai/text-davinci-002",
-            model_deployment="openai/text-davinci-002",
+            model="ai21/jamba-instruct",
+            model_deployment="ai21/jamba-instruct",
             prompt="The Center for Research on Foundation Models (CRFM) is "
             "an interdisciplinary initiative born out of the Stanford "
             "Institute for Human-Centered Artificial Intelligence (HAI) "

helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py

@@ -37,8 +37,8 @@ class TestOpenAITokenCostEstimator:
 
     def test_estimate_tokens(self):
         request = Request(
-            model="openai/text-davinci-002",
-            model_deployment="openai/text-davinci-002",
+            model="openai/davinci-002",
+            model_deployment="openai/davinci-002",
             prompt=TestOpenAITokenCostEstimator.TEST_PROMPT,
             num_completions=3,
             max_tokens=100,
@@ -49,8 +49,8 @@ class TestOpenAITokenCostEstimator:
 
     def test_estimate_tokens_with_echo_prompt(self):
         request = Request(
-            model="openai/text-davinci-002",
-            model_deployment="openai/text-davinci-002",
+            model="openai/davinci-002",
+            model_deployment="openai/davinci-002",
             prompt=TestOpenAITokenCostEstimator.TEST_PROMPT,
             echo_prompt=True,
             num_completions=1,

helm/benchmark/metrics/vision_language/image_metrics.py

@@ -35,7 +35,7 @@ try:
     from PIL import Image
     import imagehash
 except ModuleNotFoundError as e:
-    handle_module_not_found_error(e, suggestions=["image2structure"])
+    handle_module_not_found_error(e, suggestions=["image2struct"])
 
 
 def pad(small_image: Image.Image, large_image: Image.Image, axis: int) -> Image.Image:

helm/benchmark/metrics/vision_language/image_utils.py

@@ -6,7 +6,7 @@ try:
     import cv2
     from PIL.Image import Image
 except ModuleNotFoundError as e:
-    handle_module_not_found_error(e, suggestions=["image2structure"])
+    handle_module_not_found_error(e, suggestions=["image2struct"])
 
 
 def preprocess_image(image: Image) -> np.ndarray:

helm/benchmark/model_metadata_registry.py

@@ -22,9 +22,6 @@ CHATML_MODEL_TAG: str = "CHATML_MODEL_TAG"
 # OpenAI Chat format
 OPENAI_CHATGPT_MODEL_TAG: str = "OPENAI_CHATGPT_MODEL_TAG"
 
-# Mistral instruction-following format
-MISTRAL_MODEL_TAG: str = "MISTRAL_MODEL_TAG"
-
 # For Anthropic models
 ANTHROPIC_CLAUDE_1_MODEL_TAG: str = "ANTHROPIC_CLAUDE_1_MODEL_TAG"
 ANTHROPIC_CLAUDE_2_MODEL_TAG: str = "ANTHROPIC_CLAUDE_2_MODEL_TAG"
@@ -69,6 +66,9 @@ OPEN_FLAMINGO_MODEL_TAG: str = "OPEN_FLAMINGO_MODEL_TAG"
 LIMITED_FUNCTIONALITY_VLM_TAG: str = "LIMITED_FUNCTIONALITY_VLM_TAG"
 FULL_FUNCTIONALITY_VLM_TAG: str = "FULL_FUNCTIONALITY_VLM_TAG"
 
+# Deprecated models that are no longer available.
+# These are usually closed API models that have been permanently removed
+DEPRECATED_MODEL_TAG: str = "DEPRECATED_MODEL_TAG"
 
 # Frozen is set to false as the model_deployment_registry.py file
 # might populate the deployment_names field.

helm/benchmark/presentation/test_run_entry.py

@@ -16,6 +16,7 @@ class TestRunEntry:
 
     @pytest.mark.parametrize("fname", list_fnames())
     def test_read_all_specs(self, fname: str):
+        pytest.skip("Skipping slow tests")
         run_entries = read_run_entries([fname])
         for entry in run_entries.entries:
             construct_run_specs(parse_object_spec(entry.description))

helm/benchmark/run.py

@@ -1,9 +1,11 @@
 import argparse
 from dataclasses import replace
 import os
+import re
 from typing import List, Optional
 
 
+from helm.benchmark import model_metadata_registry
 from helm.benchmark.presentation.run_entry import RunEntry, read_run_entries
 from helm.common.cache_backend_config import MongoCacheBackendConfig, SqliteCacheBackendConfig
 from helm.common.general import ensure_directory_exists
@@ -314,6 +316,19 @@ def main():
     ensure_directory_exists(args.output_path)
     set_benchmark_output_path(args.output_path)
 
+    # Validate the --models-to-run flag
+    if args.models_to_run:
+        all_models = set(model_metadata_registry.get_all_models())
+        for model_to_run in args.models_to_run:
+            if model_to_run not in all_models:
+                raise Exception(f"Unknown model '{model_to_run}' passed to --models-to-run")
+    else:
+        model_expander_pattern = re.compile(
+            r"\bmodel=(?:all|text_code|text|code|instruction_following|full_functionality_text|limited_functionality_text)\b"  # noqa: E501
+        )
+        if any(model_expander_pattern.search(run_entry.description) for run_entry in run_entries):
+            raise Exception("--models-to-run must be set if the `models=` run expander expands to multiple models")
+
     run_specs = run_entries_to_run_specs(
         run_entries=run_entries,
         max_eval_instances=args.max_eval_instances,

helm/benchmark/run_expander.py

@@ -10,6 +10,7 @@ from helm.benchmark.model_metadata_registry import (
     get_all_text_models,
     get_model_metadata,
     get_model_names_with_tag,
+    DEPRECATED_MODEL_TAG,
     FULL_FUNCTIONALITY_TEXT_MODEL_TAG,
     LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG,
     ABLATION_MODEL_TAG,
@@ -343,16 +344,6 @@ class AnthropicClaude3RunExpander(RunExpander):
             run_spec,
             adapter_spec=replace(run_spec.adapter_spec, stop_sequences=stop_sequences_with_non_whitespace),
         )
-        if run_spec.adapter_spec.method == ADAPT_MULTIPLE_CHOICE_JOINT:
-            instructions = "Answer with only a single letter."
-            if run_spec.adapter_spec.instructions:
-                instructions = f"{instructions}\n\n{run_spec.adapter_spec.instructions}"
-            return [
-                replace(
-                    run_spec,
-                    adapter_spec=replace(run_spec.adapter_spec, instructions=instructions),
-                ),
-            ]
         return [run_spec]
 
 
@@ -610,6 +601,12 @@ class ModelRunExpander(ReplaceValueRunExpander):
                 values_dict["ablation"] = models
             else:
                 values_dict[family_name] = models
+
+        # For each of the keys above, filter out deprecated models.
+        deprecated_models = set(get_model_names_with_tag(DEPRECATED_MODEL_TAG))
+        for family_name in values_dict.keys():
+            values_dict[family_name] = [model for model in values_dict[family_name] if model not in deprecated_models]
+
         return values_dict
 
 
@@ -1402,23 +1399,26 @@ class OutputFormatInstructions(RunExpander):
 
     name = "output_format_instructions"
 
+    _SUFFIX_SUFFIX = "_suffix"
+
     def __init__(self, scenario: str):
-        self.scenario = scenario
+        if scenario.endswith(OutputFormatInstructions._SUFFIX_SUFFIX):
+            self.scenario = scenario[: -len(OutputFormatInstructions._SUFFIX_SUFFIX)]
+            self.suffix = True
+        else:
+            self.scenario = scenario
+            self.suffix = False
 
     def expand(self, run_spec: RunSpec) -> List[RunSpec]:
         if run_spec.adapter_spec.method == ADAPT_MULTIPLE_CHOICE_JOINT:
             if self.scenario == "mmlu_only_last_question":
                 instructions = "Answer only the last question with only a single letter."
+            elif self.scenario == "mmlu":
+                instructions = "Answer with only a single letter."
+            elif self.scenario == "mcqa":
+                instructions = "Answer with only a single letter."
             else:
                 instructions = "Answer with only a single letter."
-            if run_spec.adapter_spec.instructions:
-                instructions = f"{instructions}\n\n{run_spec.adapter_spec.instructions}"
-            return [
-                replace(
-                    run_spec,
-                    adapter_spec=replace(run_spec.adapter_spec, instructions=instructions),
-                ),
-            ]
         elif run_spec.adapter_spec.method == ADAPT_GENERATION:
             output_noun = run_spec.adapter_spec.output_prefix.split(":")[0]
             if self.scenario == "narrative_qa":
@@ -1433,27 +1433,53 @@ class OutputFormatInstructions(RunExpander):
                     instructions = f"Answer with the {output_noun.lower()}."
                 else:
                     instructions = "Answer yes or no."
+            elif self.scenario == "legalbench_abercrombie":
+                instructions = "Answer with only 'generic', 'descriptive', 'suggestive', 'arbitrary' or 'fanciful'."
+            elif self.scenario == "legalbench_function_of_decision_section":
+                instructions = "Answer with only 'Facts', 'Procedural History', 'Issue', 'Rule', 'Analysis', 'Conclusion' or 'Decree'."  # noqa: E501
+            elif self.scenario == "legalbench_yes_or_no":
+                instructions = "Answer with only 'Yes' or 'No'."
             elif self.scenario == "wmt_14":
                 instructions = "Answer with the English translation."
-            else:
-                raise ValueError(f"Unknown scenario {self.scenario}")
-
-            if run_spec.adapter_spec.output_prefix:
+            elif self.scenario == "wmt_14_only_last_sentence":
+                instructions = "Answer with only the English translation for the last sentence."
+            elif self.scenario == "math":
+                instructions = "Wrap the final answer with the \\boxed{} command."
+            elif self.scenario == "numeric_nlg":
+                instructions = "Answer with only description of the last table as a single paragraph on a single line."
+            elif self.scenario == "tab_fact":
                 instructions = (
-                    f"{instructions} Do not include '{run_spec.adapter_spec.output_prefix.strip()}' in your answer."
+                    "Answer with only the classification of the last statement, either 'refuted' or 'entailed'."
+                )
+            elif self.scenario == "wikitq":
+                instructions = (
+                    "Answer only the last question with a short answer. "
+                    "Avoid extra, unnecessary information in the answer."
                 )
-
-            if run_spec.adapter_spec.instructions:
-                instructions = f"{instructions}\n\n{run_spec.adapter_spec.instructions}"
             else:
-                instructions = f"{instructions}\n"
+                raise ValueError(f"Unknown scenario {self.scenario}")
+
+        if self.suffix:
             return [
                 replace(
                     run_spec,
-                    adapter_spec=replace(run_spec.adapter_spec, instructions=instructions),
+                    adapter_spec=replace(
+                        run_spec.adapter_spec,
+                        global_suffix=f"{run_spec.adapter_spec.global_suffix}\n\n{instructions}",
+                    ),
                 ),
             ]
-        raise ValueError(f"Unknown scenario {self.scenario}")
+
+        if run_spec.adapter_spec.instructions:
+            instructions = f"{instructions}\n\n{run_spec.adapter_spec.instructions}"
+        else:
+            instructions = f"{instructions}\n"
+        return [
+            replace(
+                run_spec,
+                adapter_spec=replace(run_spec.adapter_spec, instructions=instructions),
+            ),
+        ]
 
 
 RUN_EXPANDER_SUBCLASSES: List[Type[RunExpander]] = [