crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

{crfm_helm-0.4.0.dist-info → crfm_helm-0.5.10.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

helm/benchmark/adaptation/adapter_spec.py

@@ -1,6 +1,29 @@
 from dataclasses import dataclass, field
 from typing import List, Optional
 
+from helm.common.image_generation_parameters import ImageGenerationParameters
+from helm.common.reeval_parameters import REEvalParameters
+
+
+# Adaptation methods
+ADAPT_GENERATION: str = "generation"
+ADAPT_CHAT: str = "chat"
+ADAPT_LANGUAGE_MODELING: str = "language_modeling"
+ADAPT_MULTIPLE_CHOICE_JOINT: str = "multiple_choice_joint"
+ADAPT_MULTIPLE_CHOICE_JOINT_CHAIN_OF_THOUGHT: str = "multiple_choice_joint_chain_of_thought"
+ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL: str = "multiple_choice_separate_original"
+ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED: str = "multiple_choice_separate_calibrated"
+ADAPT_RANKING_BINARY: str = "ranking_binary"
+ADAPT_EHR_INSTRUCTION: str = "ehr_instruction"
+ADAPT_MULTIPLE_CHOICE_SEPARATE_METHODS: List[str] = [
+    ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL,
+    ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED,
+]
+
+# Multimodal adaptation methods
+ADAPT_GENERATION_MULTIMODAL: str = "generation_multimodal"
+ADAPT_MULTIPLE_CHOICE_JOINT_MULTIMODAL: str = "multiple_choice_joint_multimodal"
+
 
 @dataclass(frozen=True)
 class Substitution:
@@ -19,82 +42,110 @@ class AdapterSpec:
     Note that an `Instance` could produce many `Request`s (e.g., one for each `Reference`).
     """
 
-    # Method of adaptation
     method: str = ""
+    """The high-level strategy for converting instances into a prompt for the language model."""
 
-    # Prepend all prompts with this string.
-    # For example, it is recommended to prefix all prompts with [NLG] for UL2.
     global_prefix: str = ""
+    """The string that is prepended to the entire prompt."""
 
-    # Append all prompts with this string.
     global_suffix: str = ""
+    """The string that is appended to the entire prompt."""
 
-    # Prompt starts with instructions
     instructions: str = ""
+    """The description of the task that is included at the very beginning of the prompt."""
 
-    # What goes before the input
     input_prefix: str = "Input: "
+    """The string that is included before each input (e.g., 'Question:')."""
 
-    # What goes after the input
     input_suffix: str = "\n"
+    """The string that is included after each input (e.g., '\\n')."""
 
-    # What goes before the input (for multiple choice)
     reference_prefix: str = "A. "
+    """The string that is included before each reference (for multiple-choice questions)."""
+
+    # Set hash=False to make `AdapterSpec` hashable
+    reference_prefix_characters: Optional[List[str]] = field(default=None, hash=False)
+    """The characters that are used to identify choices for multiple-choice questions e.g. ["A", "B", "C", "D"].
+    If unset, defaults to the sequence of ascending characters starting from the first character of reference_prefix."""
 
-    # What goes before the input (for multiple choice)
     reference_suffix: str = "\n"
+    """The string that is included after each reference (for multiple-choice questions)."""
+
+    chain_of_thought_prefix: str = ""
+    """The string that is included before each chain of thought. (e.g., 'Let\'s think step by step')"""
+
+    chain_of_thought_suffix: str = "\n"
+    """The string that is included after each chain of thought. (e.g., 'The correct answer is')"""
 
-    # What goes before the output
     output_prefix: str = "Output: "
+    """The string that is included before the correct answer/predicted output (e.g., 'Answer:')."""
 
-    # What goes after the output
     output_suffix: str = "\n"
+    """The string that is included after the correct answer/predicted output (e.g., '\\n')."""
 
-    # What goes between instruction and in-context example blocks in the constructed prompt
     instance_prefix: str = "\n"
+    """The string that is included before each instance (e.g., '\\n\\n')."""
 
-    # List of regular expression substitutions that we perform
     substitutions: List[Substitution] = field(default_factory=list, hash=False)
+    """A list of regular expression substitutions (e.g., replacing '\\n' with ';\\n')
+    to perform at the very end on the prompt."""
 
-    # Maximum number of (in-context) training instances to put into the prompt
     max_train_instances: int = 5
+    """Maximum number of training instances to include in the prompt (currently by randomly sampling)."""
 
-    # Maximum number of evaluation instances. For getting valid numbers, this
-    # should be the entire dataset; only reduce this for piloting.
     max_eval_instances: Optional[int] = None
+    """Maximum number of instances to evaluate on (over all splits - test, valid, etc.)."""
 
-    # Generate this many outputs (which could be realized by `num_completions`
-    # or `top_k_per_token`).
     num_outputs: int = 5
+    """Maximum number of possible outputs to generate by sampling multiple outputs."""
 
-    # Number of trials, where in each trial we choose an independent, random
-    # set of training instances.  Used to compute error bars.
     num_train_trials: int = 1
+    """Number of trials, where in each trial we choose an independent, random set of training instances.
+    Used to compute variance."""
+
+    num_trials: int = 1
+    """Number of trials, where we query the model with the same requests, but different random seeds."""
 
-    # If true, randomly sample N training examples; if false, select N consecutive training examples
     sample_train: bool = True
+    """If true, randomly sample N training examples; if false, select N consecutive training examples"""
 
     # Decoding parameters (inherited by `Request`)
 
-    # Model deployment to make the request to (need to fill in)
     model_deployment: str = ""
+    """Name of the language model deployment (<host_organization>/<model name>) to send requests to."""
 
-    # DEPRECATED: old model field, kept for backward compatibility
-    # TODO: Remove this once we do not wish to support backward compatibility anymore.
     model: str = ""
+    """Name of the language model (<creator_organization>/<model name>) to send requests to."""
 
-    # Temperature to use
     temperature: float = 1
+    """Temperature parameter used in generation."""
 
-    # Maximum number of tokens to generate
     max_tokens: int = 100
+    """Maximum number of tokens to generate."""
 
-    # When to stop (set hash=False to make `AdapterSpec` hashable)
+    # Set hash=False to make `AdapterSpec` hashable
     stop_sequences: List[str] = field(default_factory=list, hash=False)
+    """List of stop sequences. Output generation will be stopped if any stop sequence is encountered."""
 
     # Random string (used concretely to bypass cache / see diverse results)
     random: Optional[str] = None
+    """Random seed (string), which guarantees reproducibility."""
 
-    # If true, for instances with multiple correct reference, the gold answer should be considered
-    # to be all of the correct references rather than any of the correct references.
     multi_label: bool = False
+    """If true, for instances with multiple correct reference, the gold answer should be considered to be all
+    of the correct references rather than any of the correct references."""
+
+    image_generation_parameters: Optional[ImageGenerationParameters] = None
+    """Parameters for image generation."""
+
+    reeval_parameters: Optional[REEvalParameters] = None
+    """Parameters for reeval evaluation."""
+
+    # Set hash=False to make `AdapterSpec` hashable
+    eval_splits: Optional[List[str]] = field(default=None, hash=False)
+    """The splits from which evaluation instances will be drawn."""
+
+    output_mapping_pattern: Optional[str] = None
+    """Pattern to apply to the output before applying the output mapping for the joint multiple choice adapter.
+    If the pattern has no group, the output mapping will be applied to the first match.
+    If the pattern has a group, the output mapping will be applied to the group of the first match."""

helm/benchmark/adaptation/adapters/adapter.py

@@ -2,7 +2,7 @@ from abc import ABC, abstractmethod
 from typing import List
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.scenarios.scenario import Instance
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
 from helm.benchmark.window_services.window_service import WindowService
@@ -22,7 +22,7 @@ class Adapter(ABC):
         )
 
     @abstractmethod
-    def adapt(self, instances: List[Instance], parallelism: int) -> ScenarioState:
+    def adapt(self, instances: List[Instance], parallelism: int) -> List[RequestState]:
         """
         Takes a a list of `Instance`s and returns a `ScenarioState` with the
         list of corresponding `RequestState`s.

helm/benchmark/adaptation/adapters/adapter_factory.py

@@ -1,31 +1,34 @@
-from typing import List
-
-from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from helm.benchmark.window_services.tokenizer_service import TokenizerService
-from .adapter import Adapter
-from .generation_adapter import GenerationAdapter
-from .language_modeling_adapter import LanguageModelingAdapter
-from .multiple_choice_joint_adapter import MultipleChoiceJointAdapter
-from .multiple_choice_separate_adapter import MultipleChoiceSeparateAdapter
-from .multiple_choice_calibrated_adapter import MultipleChoiceCalibratedAdapter
-from .binary_ranking_adapter import BinaryRankingAdapter
-from .multimodal.generation_multimodal_adapter import GenerationMultimodalAdapter
-
-# Adaptation methods
-ADAPT_GENERATION: str = "generation"
-ADAPT_LANGUAGE_MODELING: str = "language_modeling"
-ADAPT_MULTIPLE_CHOICE_JOINT: str = "multiple_choice_joint"
-ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL: str = "multiple_choice_separate_original"
-ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED: str = "multiple_choice_separate_calibrated"
-ADAPT_RANKING_BINARY: str = "ranking_binary"
-
-ADAPT_MULTIPLE_CHOICE_SEPARATE_METHODS: List[str] = [
-    ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL,
+from helm.benchmark.adaptation.adapter_spec import (
+    ADAPT_EHR_INSTRUCTION,
+    ADAPT_GENERATION,
+    ADAPT_CHAT,
+    ADAPT_GENERATION_MULTIMODAL,
+    ADAPT_LANGUAGE_MODELING,
+    ADAPT_MULTIPLE_CHOICE_JOINT,
+    ADAPT_MULTIPLE_CHOICE_JOINT_CHAIN_OF_THOUGHT,
+    ADAPT_MULTIPLE_CHOICE_JOINT_MULTIMODAL,
     ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED,
-]
-
-# Multimodal adaptation methods
-ADAPT_GENERATION_MULTIMODAL: str = "generation_multimodal"
+    ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL,
+    ADAPT_RANKING_BINARY,
+    AdapterSpec,
+)
+from helm.benchmark.adaptation.adapters.adapter import Adapter
+from helm.benchmark.adaptation.adapters.binary_ranking_adapter import BinaryRankingAdapter
+from helm.benchmark.adaptation.adapters.generation_adapter import GenerationAdapter
+from helm.benchmark.adaptation.adapters.chat_adapter import ChatAdapter
+from helm.benchmark.adaptation.adapters.language_modeling_adapter import LanguageModelingAdapter
+from helm.benchmark.adaptation.adapters.multimodal.generation_multimodal_adapter import GenerationMultimodalAdapter
+from helm.benchmark.adaptation.adapters.multimodal.multiple_choice_joint_multimodal_adapter import (
+    MultipleChoiceJointMultimodalAdapter,
+)
+from helm.benchmark.adaptation.adapters.multiple_choice_calibrated_adapter import MultipleChoiceCalibratedAdapter
+from helm.benchmark.adaptation.adapters.multiple_choice_joint_adapter import MultipleChoiceJointAdapter
+from helm.benchmark.adaptation.adapters.multiple_choice_joint_chain_of_thought_adapter import (
+    MultipleChoiceJointChainOfThoughtAdapter,
+)
+from helm.benchmark.adaptation.adapters.multiple_choice_separate_adapter import MultipleChoiceSeparateAdapter
+from helm.benchmark.window_services.tokenizer_service import TokenizerService
+from helm.benchmark.adaptation.adapters.ehr_instruction_adapter import EHRInstructionAdapter
 
 
 class AdapterFactory:
@@ -37,12 +40,18 @@ class AdapterFactory:
         method: str = adapter_spec.method
         adapter: Adapter
 
-        if method == ADAPT_GENERATION:
+        if method == ADAPT_EHR_INSTRUCTION:
+            adapter = EHRInstructionAdapter(adapter_spec, tokenizer_service)
+        elif method == ADAPT_GENERATION:
             adapter = GenerationAdapter(adapter_spec, tokenizer_service)
+        elif method == ADAPT_CHAT:
+            adapter = ChatAdapter(adapter_spec, tokenizer_service)
         elif method == ADAPT_LANGUAGE_MODELING:
             adapter = LanguageModelingAdapter(adapter_spec, tokenizer_service)
         elif method == ADAPT_MULTIPLE_CHOICE_JOINT:
             adapter = MultipleChoiceJointAdapter(adapter_spec, tokenizer_service)
+        elif method == ADAPT_MULTIPLE_CHOICE_JOINT_CHAIN_OF_THOUGHT:
+            adapter = MultipleChoiceJointChainOfThoughtAdapter(adapter_spec, tokenizer_service)
         elif method == ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL:
             adapter = MultipleChoiceSeparateAdapter(adapter_spec, tokenizer_service)
         elif method == ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED:
@@ -51,6 +60,8 @@ class AdapterFactory:
             adapter = BinaryRankingAdapter(adapter_spec, tokenizer_service)
         elif method == ADAPT_GENERATION_MULTIMODAL:
             adapter = GenerationMultimodalAdapter(adapter_spec, tokenizer_service)
+        elif method == ADAPT_MULTIPLE_CHOICE_JOINT_MULTIMODAL:
+            adapter = MultipleChoiceJointMultimodalAdapter(adapter_spec, tokenizer_service)
         else:
             raise ValueError(f"Invalid adaptation method: {method}")
 

helm/benchmark/adaptation/adapters/binary_ranking_adapter.py

@@ -3,7 +3,7 @@ from typing import List, Optional
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.scenarios.scenario import Instance, Reference, TRAIN_SPLIT, EVAL_SPLITS, CORRECT_TAG
 from helm.common.request import Request
-from .in_context_learning_adapter import InContextLearningAdapter
+from helm.benchmark.adaptation.adapters.in_context_learning_adapter import InContextLearningAdapter
 
 
 class BinaryRankingAdapter(InContextLearningAdapter):

helm/benchmark/adaptation/adapters/chat_adapter.py

@@ -0,0 +1,49 @@
+from typing import List
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.scenarios.scenario import Instance
+from helm.common.request import Request
+from helm.benchmark.adaptation.adapters.in_context_learning_adapter import InContextLearningAdapter
+
+
+class ChatAdapter(InContextLearningAdapter):
+    """
+    Each `Instance` in a `Scenario` has a history of the format:
+
+    [
+        {"role": "user", "content": <user-content>},
+        {"role": "assistant", "content": <assistant-content>},
+        {"role": "user", "content": <user-content>},
+        ...
+    ]
+
+    """
+
+    def generate_requests(
+        self, eval_instance: Instance, train_trial_index: int, training_instances: List[Instance]
+    ) -> List[RequestState]:
+        if eval_instance.input.messages is None:
+            raise ValueError("ChatAdapter requires input.messages of instances to be non-empty")
+        request = Request(
+            model=self.adapter_spec.model,
+            model_deployment=self.adapter_spec.model_deployment,
+            messages=eval_instance.input.messages,
+            num_completions=self.adapter_spec.num_outputs,
+            temperature=self.adapter_spec.temperature,
+            max_tokens=self.adapter_spec.max_tokens,
+            stop_sequences=self.adapter_spec.stop_sequences,
+            random=self.adapter_spec.random,
+            image_generation_parameters=self.adapter_spec.image_generation_parameters,
+        )
+        request_state = RequestState(
+            instance=eval_instance,
+            reference_index=None,
+            request_mode=None,
+            train_trial_index=train_trial_index,
+            output_mapping=None,
+            request=request,
+            result=None,
+            num_train_instances=0,
+            prompt_truncated=False,
+        )
+        return [request_state]

helm/benchmark/adaptation/adapters/ehr_instruction_adapter.py

@@ -0,0 +1,108 @@
+from typing import List, Optional
+
+from helm.benchmark.adaptation.adapters.generation_adapter import GenerationAdapter
+from helm.benchmark.adaptation.prompt import Prompt
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.scenarios.scenario import TRAIN_SPLIT, Instance
+from helm.benchmark.window_services.window_service import EncodeResult
+from helm.common.tokenization_request import TokenizationToken
+
+
+# in the prompt templates for EHR instructions, this is the placeholder for the EHR part
+# which we use to compute accurate tokenized sequence lengths
+PROMPT_TEMPLATE_EHR_PLACEHOLDER = "{ehr}"
+
+
+class EHRInstructionAdapter(GenerationAdapter):
+    """
+    Each instance consists of the following:
+
+    EHRInstructionInput:
+        question: the question to answer or instruction to follow
+        ehr: the XML-tagged EHR to use as context to answer the question
+        prompt_template: a string template for how to combine the question + ehr
+
+    Reference output:
+        text: the 'golden' clinician response to the question
+
+    This Adapter combines the above into RequestStates with logic to truncate the EHR specifically
+    to fit in the context window with enough room for the instruction/question and the specified
+    amount of generated tokens.
+    """
+
+    def adapt(self, instances: List[Instance], parallelism: int) -> List[RequestState]:
+        """
+        Main adaptation method which takes all instances and turns them into `RequestState` objects.
+        """
+        # sanity check, since for now we assume that there are no training instances at all
+        if any(instance.split == TRAIN_SPLIT for instance in instances):
+            raise RuntimeError(f"Got train instances for {self.__class__.__name__} - expected only eval instances.")
+
+        # use superclass implementation here
+        return super().adapt(instances, parallelism)
+
+    def construct_prompt(
+        self,
+        train_instances: List[Instance],  # unused
+        eval_instance: Instance,
+        include_output: bool,  # unused
+        reference_index: Optional[int],  # unused
+    ) -> Prompt:
+        """
+        Uses the instance to construct a prompt for a given eval instance.
+
+        Parameters
+        ----------
+        eval_instance: Instance
+            the instance we wish to use to construct the prompt
+        """
+        # start by simply getting the inputs
+        question = eval_instance.input.text
+        assert eval_instance.extra_data is not None
+        ehr_text: str = eval_instance.extra_data["ehr"]
+        prompt_template: str = eval_instance.extra_data["prompt_template"]
+        full_prompt_text = prompt_template.format(question=question, ehr=ehr_text)
+
+        # insert the question and see how many tokens we have so far
+        prompt_with_instr_no_ehr_placeholder = prompt_template.format(question=question, ehr="")
+        num_tokens_no_ehr = self.window_service.get_num_tokens(prompt_with_instr_no_ehr_placeholder)
+
+        # number of tokens we can allow the EHR part to be
+        target_ehr_num_tokens = (
+            self.window_service.max_request_length - self.adapter_spec.max_tokens - num_tokens_no_ehr
+        )
+
+        # round-trip tokenization to get the correct token length we need
+        # NOTE: we truncate from the left side so that the most recent pieces of the EHR are included in the context
+        # as opposed to the canonical way of truncating from the right. This is done to match the MedAlign method.
+        full_ehr_tokens: EncodeResult = self.window_service.encode(ehr_text, max_length=None, truncation=False)
+        truncated_ehr_tokens: List[TokenizationToken] = full_ehr_tokens.tokens[-target_ehr_num_tokens:]
+        ehr_truncated: str
+        ehr_truncated = self.window_service.decode(truncated_ehr_tokens)
+
+        # create the truncated prompt
+        truncated_prompt_text = prompt_template.format(question=question, ehr=ehr_truncated)
+        num_truncations = 1
+        while (
+            num_extra_tokens := self.adapter_spec.max_tokens
+            + self.window_service.get_num_tokens(truncated_prompt_text)
+            - self.window_service.max_request_length
+        ) > 0:
+            truncated_ehr_tokens = truncated_ehr_tokens[num_extra_tokens:]
+            ehr_truncated = self.window_service.decode(truncated_ehr_tokens)
+            truncated_prompt_text = prompt_template.format(question=question, ehr=ehr_truncated)
+            num_truncations += 1
+
+        # naively construct the full non-truncated prompt
+        prompt = Prompt(
+            global_prefix=self.adapter_spec.global_prefix,
+            global_suffix=self.adapter_spec.global_suffix,
+            instance_prefix=self.adapter_spec.instance_prefix,
+            substitutions=self.adapter_spec.substitutions,
+            instructions_block=self.adapter_spec.instructions,
+            train_instance_blocks=[],
+            eval_instance_block=full_prompt_text,
+            truncated_text=truncated_prompt_text,
+        )
+
+        return prompt

helm/benchmark/adaptation/adapters/generation_adapter.py

@@ -4,7 +4,7 @@ from helm.benchmark.adaptation.prompt import Prompt
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.scenarios.scenario import Instance
 from helm.common.request import Request
-from .in_context_learning_adapter import InContextLearningAdapter
+from helm.benchmark.adaptation.adapters.in_context_learning_adapter import InContextLearningAdapter
 
 
 class GenerationAdapter(InContextLearningAdapter):
@@ -46,6 +46,7 @@ class GenerationAdapter(InContextLearningAdapter):
             max_tokens=self.adapter_spec.max_tokens,
             stop_sequences=self.adapter_spec.stop_sequences,
             random=self.adapter_spec.random,
+            image_generation_parameters=self.adapter_spec.image_generation_parameters,
         )
         request_state = RequestState(
             instance=eval_instance,

helm/benchmark/adaptation/adapters/in_context_learning_adapter.py

@@ -7,11 +7,11 @@ from typing import List, Dict, Optional
 
 from helm.benchmark.adaptation.prompt import Prompt
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.scenarios.scenario import Instance, TRAIN_SPLIT, EVAL_SPLITS, Reference
 from helm.common.general import parallel_map
-from helm.common.hierarchical_logger import hlog, htrack, htrack_block
-from .adapter import Adapter
+from helm.common.request import Request
+from helm.common.hierarchical_logger import hlog, htrack, htrack_block, hwarn
+from helm.benchmark.adaptation.adapters.adapter import Adapter
 
 
 class InContextLearningAdapter(Adapter, ABC):
@@ -30,7 +30,7 @@ class InContextLearningAdapter(Adapter, ABC):
         pass
 
     @htrack(None)
-    def adapt(self, instances: List[Instance], parallelism: int) -> ScenarioState:
+    def adapt(self, instances: List[Instance], parallelism: int) -> List[RequestState]:
         """
         Takes a list of `Instance`s and builds a list of corresponding `RequestState`s.
         The reason we don't do this per eval instance is that we create a common set of
@@ -39,8 +39,8 @@ class InContextLearningAdapter(Adapter, ABC):
         # Pick out training instances
         all_train_instances: List[Instance] = [instance for instance in instances if instance.split == TRAIN_SPLIT]
         if len(all_train_instances) < self.adapter_spec.max_train_instances:
-            hlog(
-                f"WARNING: only {len(all_train_instances)} training instances, "
+            hwarn(
+                f"only {len(all_train_instances)} training instances, "
                 f"wanted {self.adapter_spec.max_train_instances}"
             )
 
@@ -64,7 +64,7 @@ class InContextLearningAdapter(Adapter, ABC):
                 )
 
         hlog(f"{len(all_request_states)} requests")
-        return ScenarioState(self.adapter_spec, all_request_states)
+        return all_request_states
 
     def _adapt_trial_index(
         self,
@@ -101,7 +101,23 @@ class InContextLearningAdapter(Adapter, ABC):
                             hlog(line)
 
         # Flatten and return
-        return [request_state for result in results for request_state in result]
+        all_request_states: List[RequestState] = [request_state for result in results for request_state in result]
+        return self._add_trials(all_request_states)
+
+    def _add_trials(self, request_states: List[RequestState]) -> List[RequestState]:
+        """Expand the request states by adding trials."""
+        if self.adapter_spec.num_trials <= 1:
+            return request_states
+
+        all_request_states: List[RequestState] = request_states.copy()
+        for i in range(1, self.adapter_spec.num_trials):
+            seed: str = str(i)
+            for request_state in request_states:
+                request: Request = replace(request_state.request, random=seed)
+                all_request_states.append(replace(request_state, request=request))
+
+        assert len(all_request_states) == len(request_states) * self.adapter_spec.num_trials
+        return all_request_states
 
     def sample_examples(
         self, all_train_instances: List[Instance], seed: int, sample_train: bool = True

helm/benchmark/adaptation/adapters/language_modeling_adapter.py

@@ -1,14 +1,13 @@
 from typing import List, Tuple, Optional
 
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.scenarios.scenario import Instance, EVAL_SPLITS
 from helm.benchmark.window_services.window_service import EncodeResult
 from helm.common.general import flatten_list, parallel_map
 from helm.common.hierarchical_logger import hlog, htrack
 from helm.common.request import Request
 from helm.common.tokenization_request import TokenizationToken
-from .adapter import Adapter
+from helm.benchmark.adaptation.adapters.adapter import Adapter
 
 
 class LanguageModelingAdapter(Adapter):
@@ -26,7 +25,7 @@ class LanguageModelingAdapter(Adapter):
     """
 
     @htrack(None)
-    def adapt(self, instances: List[Instance], parallelism: int) -> ScenarioState:
+    def adapt(self, instances: List[Instance], parallelism: int) -> List[RequestState]:
         """
         Takes a list of `Instance`s and builds a list of corresponding `RequestState`s.
         Only requires eval instances.
@@ -46,7 +45,7 @@ class LanguageModelingAdapter(Adapter):
         )
         hlog(f"{len(all_request_states)} requests")
 
-        return ScenarioState(self.adapter_spec, all_request_states)
+        return all_request_states
 
     def _generate_requests(self, eval_instance: Instance) -> List[RequestState]:
         """

helm/benchmark/adaptation/adapters/multimodal/generation_multimodal_adapter.py

@@ -3,8 +3,10 @@ from typing import List
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.scenarios.scenario import Instance
 from helm.common.request import Request
-from .in_context_learning_multimodal_adapter import InContextLearningMultimodalAdapter
-from .multimodal_prompt import MultimodalPrompt
+from helm.benchmark.adaptation.adapters.multimodal.in_context_learning_multimodal_adapter import (
+    InContextLearningMultimodalAdapter,
+)
+from helm.benchmark.adaptation.adapters.multimodal.multimodal_prompt import MultimodalPrompt
 
 
 class GenerationMultimodalAdapter(InContextLearningMultimodalAdapter):

helm/benchmark/adaptation/adapters/multimodal/in_context_learning_multimodal_adapter.py

@@ -8,7 +8,7 @@ from helm.common.hierarchical_logger import hlog
 from helm.common.media_object import MediaObject, MultimediaObject
 from helm.common.request import Request
 from helm.benchmark.adaptation.adapters.in_context_learning_adapter import InContextLearningAdapter
-from .multimodal_prompt import MultimodalPrompt
+from helm.benchmark.adaptation.adapters.multimodal.multimodal_prompt import MultimodalPrompt
 
 
 class InContextLearningMultimodalAdapter(InContextLearningAdapter, ABC):
@@ -79,6 +79,7 @@ class InContextLearningMultimodalAdapter(InContextLearningAdapter, ABC):
         # Prompt
         prompt = MultimodalPrompt(
             global_prefix=self.adapter_spec.global_prefix,
+            global_suffix=self.adapter_spec.global_suffix,
             instructions=self.adapter_spec.instructions,
             train_instance_blocks=train_instance_blocks,
             eval_instance_block=eval_instance_block,

helm/benchmark/adaptation/adapters/multimodal/multimodal_prompt.py

@@ -11,6 +11,9 @@ class MultimodalPrompt:
     # Global prefix, carried over from `AdapterSpec`
     global_prefix: str
 
+    # Global suffix, carried over from `AdapterSpec`
+    global_suffix: str
+
     # Instance prefix, carried over from `AdapterSpec`. What goes between the instruction and instances.
     instance_prefix: str
 
@@ -47,6 +50,10 @@ class MultimodalPrompt:
         if self.global_prefix:
             result = result.add_textual_prefix(self.global_prefix)
 
+        # Add the global prefix if one exists
+        if self.global_suffix:
+            result = result.add_textual_suffix(self.global_suffix)
+
         return result
 
     @property