crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/adaptation/adapters/multimodal/multiple_choice_joint_multimodal_adapter.py

@@ -0,0 +1,112 @@
+from abc import ABC
+from typing import Dict, List, Optional
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.scenarios.scenario import Instance
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.request import Request
+from helm.benchmark.adaptation.adapters.multimodal.in_context_learning_multimodal_adapter import (
+    InContextLearningMultimodalAdapter,
+)
+from helm.benchmark.adaptation.adapters.multimodal.multimodal_prompt import MultimodalPrompt
+
+
+class MultipleChoiceJointMultimodalAdapter(InContextLearningMultimodalAdapter, ABC):
+    """
+    An `Adapter`, guided by the `AdapterSpec`, takes a `Scenario` and produces
+    a `ScenarioState`. This `Adapter` has additional logic to support in-context
+    learning for multimodal models.
+    """
+
+    def get_prefix_char(self, prefix: str) -> str:
+        return [char for char in prefix if char.isalnum()][0]
+
+    def get_reference_prefix(self, prefix: str, i: int) -> str:
+        """
+        Example: prefix = "\nA. ", i = 2, return "\nC. "
+        """
+        old_prefix_char = self.get_prefix_char(prefix)
+        new_prefix_char = (
+            self.adapter_spec.reference_prefix_characters[i]
+            if self.adapter_spec.reference_prefix_characters
+            else chr(ord(old_prefix_char) + i)
+        )
+        return prefix.replace(old_prefix_char, new_prefix_char)
+
+    def generate_requests(
+        self, eval_instance: Instance, train_trial_index: int, training_instances: List[Instance]
+    ) -> List[RequestState]:
+        prompt: MultimodalPrompt = self.construct_prompt(
+            training_instances, eval_instance, include_output=False, reference_index=None
+        )
+        output_mapping: Dict[str, str] = dict(
+            (self.get_reference_prefix("A", reference_index), reference.output.text)
+            for reference_index, reference in enumerate(eval_instance.references)
+        )
+        request = Request(
+            model=self.adapter_spec.model,
+            model_deployment=self.adapter_spec.model_deployment,
+            multimodal_prompt=prompt.multimedia_object,
+            num_completions=self.adapter_spec.num_outputs,
+            temperature=self.adapter_spec.temperature,
+            max_tokens=self.adapter_spec.max_tokens,
+            stop_sequences=[],
+            random=self.adapter_spec.random,
+        )
+        request_state = RequestState(
+            instance=eval_instance,
+            reference_index=None,
+            request_mode=None,
+            train_trial_index=train_trial_index,
+            output_mapping=output_mapping,
+            request=request,
+            result=None,
+            num_train_instances=prompt.num_train_instances,
+            prompt_truncated=False,
+        )
+        return [request_state]
+
+    def construct_example_multimodal_prompt(
+        self, instance: Instance, include_output: bool, reference_index: Optional[int]
+    ) -> MultimediaObject:
+        """
+        Returns a single example of the prompt. `include_output` controls whether the gold output is included.
+        """
+        # Input
+        assert instance.input.multimedia_content is not None
+        result: MultimediaObject = instance.input.multimedia_content.add_textual_prefix(self.adapter_spec.input_prefix)
+        result = result.add_textual_suffix(self.adapter_spec.input_suffix)
+
+        # Include the references
+        delimiter: str = ", "
+        no_correct_references: str = "n/a"
+        output: str = no_correct_references
+        for reference_index, reference in enumerate(instance.references):
+            prefix = self.get_reference_prefix(self.adapter_spec.reference_prefix, reference_index)
+
+            if reference.output.multimedia_content is not None:
+                reference_output_content: MultimediaObject = reference.output.multimedia_content
+                reference_output_content = reference_output_content.add_textual_prefix(prefix)
+                reference_output_content = reference_output_content.add_textual_suffix(
+                    self.adapter_spec.reference_suffix
+                )
+                result = result.combine(reference_output_content)
+            else:
+                result = result.add_textual_suffix(prefix + reference.output.text + self.adapter_spec.reference_suffix)
+
+            if reference.is_correct:
+                if output == no_correct_references:
+                    output = self.get_reference_prefix("A", reference_index)
+                elif self.adapter_spec.multi_label:
+                    output += delimiter
+                    output += self.get_reference_prefix("A", reference_index)
+
+        if include_output:
+            output_content: MultimediaObject = MultimediaObject([MediaObject(text=output, content_type="text/plain")])
+            output_content = output_content.add_textual_prefix(self.adapter_spec.output_prefix)
+            output_content = output_content.add_textual_suffix(self.adapter_spec.output_suffix)
+            result = result.combine(output_content)
+        else:
+            result = result.add_textual_suffix(self.adapter_spec.output_prefix.rstrip())
+
+        return result

helm/benchmark/adaptation/adapters/multimodal/test_in_context_learning_multimodal_adapter.py

@@ -1,20 +1,23 @@
 import shutil
 import tempfile
 import unittest
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
 
 from helm.common.media_object import MediaObject, MultimediaObject
 from helm.benchmark.scenarios.scenario import Instance, Reference, Input, Output, TEST_SPLIT, TRAIN_SPLIT, CORRECT_TAG
 from helm.benchmark.window_services.test_utils import get_tokenizer_service
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_GENERATION_MULTIMODAL, AdapterFactory
-from .in_context_learning_multimodal_adapter import InContextLearningMultimodalAdapter
-from .multimodal_prompt import MultimodalPrompt
+from helm.benchmark.adaptation.adapters.multimodal.in_context_learning_multimodal_adapter import (
+    InContextLearningMultimodalAdapter,
+)
+from helm.benchmark.adaptation.adapters.multimodal.multimodal_prompt import MultimodalPrompt
 
 
 class TestInContextLearningMultimodalAdapter(unittest.TestCase):
     def setup_method(self, _):
         self._path: str = tempfile.mkdtemp()
-        self._tokenizer_service = get_tokenizer_service(self._path)
+        self._tokenizer_service = get_tokenizer_service(self._path, BlackHoleCacheBackendConfig())
 
     def teardown_method(self, _):
         shutil.rmtree(self._path)

helm/benchmark/adaptation/adapters/multimodal/test_multimodal_prompt.py

@@ -2,7 +2,7 @@ from typing import List
 import unittest
 
 from helm.common.media_object import MediaObject, MultimediaObject
-from .multimodal_prompt import MultimodalPrompt
+from helm.benchmark.adaptation.adapters.multimodal.multimodal_prompt import MultimodalPrompt
 
 
 class TestMultimodalContent(unittest.TestCase):
@@ -32,6 +32,7 @@ class TestMultimodalContent(unittest.TestCase):
 
         prompt = MultimodalPrompt(
             global_prefix="[START]",
+            global_suffix="",
             instance_prefix="\n",
             instructions="Please answer the following questions about the images.",
             train_instance_blocks=train_instance_blocks,
@@ -67,6 +68,7 @@ class TestMultimodalContent(unittest.TestCase):
 
         prompt = MultimodalPrompt(
             global_prefix="",
+            global_suffix="",
             instance_prefix="\n",
             instructions="",
             train_instance_blocks=[],

helm/benchmark/adaptation/adapters/multiple_choice_calibrated_adapter.py

@@ -3,7 +3,7 @@ from typing import List
 
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.scenarios.scenario import Instance, Input
-from .multiple_choice_separate_adapter import MultipleChoiceSeparateAdapter
+from helm.benchmark.adaptation.adapters.multiple_choice_separate_adapter import MultipleChoiceSeparateAdapter
 
 
 class MultipleChoiceCalibratedAdapter(MultipleChoiceSeparateAdapter):

helm/benchmark/adaptation/adapters/multiple_choice_joint_adapter.py

@@ -3,7 +3,7 @@ from typing import List, Dict, Optional
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.scenarios.scenario import Instance
 from helm.common.request import Request
-from .in_context_learning_adapter import InContextLearningAdapter
+from helm.benchmark.adaptation.adapters.in_context_learning_adapter import InContextLearningAdapter
 
 
 class MultipleChoiceJointAdapter(InContextLearningAdapter):
@@ -35,22 +35,31 @@ class MultipleChoiceJointAdapter(InContextLearningAdapter):
 
     In general, each example is:
 
-        <input_prefix><input><reference_prefixes[0]><reference><output_prefix><output>
+        <input_prefix><input><reference_prefixes[index]><reference><output_prefix><output>
     """
 
-    @staticmethod
-    def get_reference_prefix(prefix: str, i: int) -> str:
+    def get_prefix_char(self, prefix: str) -> str:
+        return [char for char in prefix if char.isalnum()][0]
+
+    def get_reference_prefix(self, prefix: str, i: int) -> str:
         """
         Example: prefix = "\nA. ", i = 2, return "\nC. "
         """
-        return prefix.replace("A", chr(ord("A") + i))
+        old_prefix_char = self.get_prefix_char(prefix)
+        new_prefix_char = (
+            self.adapter_spec.reference_prefix_characters[i]
+            if self.adapter_spec.reference_prefix_characters
+            else chr(ord(old_prefix_char) + i)
+        )
+        return prefix.replace(old_prefix_char, new_prefix_char)
 
     def generate_requests(
         self, eval_instance: Instance, train_trial_index: int, training_instances: List[Instance]
     ) -> List[RequestState]:
+        prefix_char = self.get_prefix_char(self.adapter_spec.reference_prefix)
         prompt = self.construct_prompt(training_instances, eval_instance, include_output=False, reference_index=None)
         output_mapping: Dict[str, str] = dict(
-            (self.get_reference_prefix("A", reference_index), reference.output.text)
+            (self.get_reference_prefix(prefix_char, reference_index), reference.output.text)
             for reference_index, reference in enumerate(eval_instance.references)
         )
         request = Request(
@@ -85,16 +94,17 @@ class MultipleChoiceJointAdapter(InContextLearningAdapter):
         # Include the references
         delimiter = ", "
         no_correct_references = "n/a"
+        prefix_char = self.get_prefix_char(self.adapter_spec.reference_prefix)
         output = no_correct_references
         for reference_index, reference in enumerate(instance.references):
             prefix = self.get_reference_prefix(self.adapter_spec.reference_prefix, reference_index)
             result += prefix + reference.output.text + self.adapter_spec.reference_suffix
             if reference.is_correct:
                 if output == no_correct_references:
-                    output = self.get_reference_prefix("A", reference_index)
+                    output = self.get_reference_prefix(prefix_char, reference_index)
                 elif self.adapter_spec.multi_label:
                     output += delimiter
-                    output += self.get_reference_prefix("A", reference_index)
+                    output += self.get_reference_prefix(prefix_char, reference_index)
 
         if include_output:
             result += self.adapter_spec.output_prefix + output + self.adapter_spec.output_suffix

helm/benchmark/adaptation/adapters/multiple_choice_joint_chain_of_thought_adapter.py

@@ -0,0 +1,87 @@
+from typing import Optional
+
+from helm.benchmark.scenarios.scenario import Instance
+from helm.benchmark.adaptation.adapters.multiple_choice_joint_adapter import MultipleChoiceJointAdapter
+
+
+class MultipleChoiceJointChainOfThoughtAdapter(MultipleChoiceJointAdapter):
+    """
+    Each `Instance` in a `Scenario` looks like this:
+
+        <input> -> <reference1>
+                   <reference2>
+                   <reference3> [correct]
+                   <reference4>
+
+        <instance_chain_of_thought>
+
+    We can define a label (e.g., letter) for each reference:
+
+        <global_prefix>
+        <instructions>
+        <input_prefix>
+        <input>                  # train
+        <input_suffix>
+        A. <reference1>
+        B. <reference2>
+        C. <reference3>
+        D. <reference4>
+        <output_prefix>
+        <chain_of_thought_prefix>
+        <instance_chain_of_thought>
+        <chain_of_thought_suffix>
+        <output>
+        <output_suffix>
+
+        <input_prefix>
+        <input>                  # test
+        <input_suffix>
+        A. <reference1>
+        B. <reference2>
+        C. <reference3>
+        D. <reference4>
+        <output_prefix>
+        <chain_of_thought_prefix>
+        <instance_chain_of_thought>
+        <chain_of_thought_suffix>
+        <output>
+        <output_suffix>
+        <global_suffix>
+
+    In general, each example is:
+
+        <input_prefix><input><input_suffix><reference_prefixes[index]><reference> \
+        <output_prefix><chain_of_thought_prefix><chain_of_thought><chain_of_thought_suffix><output><output_suffix>
+    """
+
+    def construct_example_prompt(self, instance: Instance, include_output: bool, reference_index: Optional[int]) -> str:
+        """Return a list of lines corresponding to this example (part of the prompt)."""
+        # Input
+        result: str = self.adapter_spec.input_prefix + instance.input.text + self.adapter_spec.input_suffix
+
+        # Include the references
+        delimiter = ", "
+        no_correct_references = "n/a"
+        output = no_correct_references
+        for reference_index, reference in enumerate(instance.references):
+            prefix = self.get_reference_prefix(self.adapter_spec.reference_prefix, reference_index)
+            result += prefix + reference.output.text + self.adapter_spec.reference_suffix
+            if reference.is_correct:
+                if output == no_correct_references:
+                    output = self.get_reference_prefix(self.adapter_spec.reference_prefix, reference_index)
+                elif self.adapter_spec.multi_label:
+                    output += delimiter
+                    output += self.get_reference_prefix(self.adapter_spec.reference_prefix, reference_index)
+
+        if include_output:
+            chain_of_thought = instance.extra_data.get("chain_of_thought", "") if instance.extra_data else ""
+            chain_of_thought_block = (
+                self.adapter_spec.chain_of_thought_prefix + chain_of_thought + self.adapter_spec.chain_of_thought_suffix
+            )
+            result += (
+                self.adapter_spec.output_prefix + chain_of_thought_block + output + self.adapter_spec.output_suffix
+            )
+        else:
+            result += self.adapter_spec.output_prefix.rstrip()
+
+        return result

helm/benchmark/adaptation/adapters/multiple_choice_separate_adapter.py

@@ -4,7 +4,7 @@ from helm.benchmark.adaptation.prompt import Prompt
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.scenarios.scenario import Instance
 from helm.common.request import Request
-from .in_context_learning_adapter import InContextLearningAdapter
+from helm.benchmark.adaptation.adapters.in_context_learning_adapter import InContextLearningAdapter
 
 
 class MultipleChoiceSeparateAdapter(InContextLearningAdapter):

helm/benchmark/adaptation/adapters/test_adapter.py

@@ -1,8 +1,9 @@
 import shutil
 import tempfile
 
-from helm.common.authentication import Authentication
-from helm.proxy.services.server_service import ServerService
+
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
+from helm.common.local_context import LocalContext
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
 
 
@@ -13,8 +14,8 @@ class TestAdapter:
 
     def setup_method(self):
         self.path: str = tempfile.mkdtemp()
-        service = ServerService(base_path=self.path, root_mode=True)
-        self.tokenizer_service = TokenizerService(service, Authentication("test"))
+        context = LocalContext(base_path=self.path, cache_backend_config=BlackHoleCacheBackendConfig())
+        self.tokenizer_service = TokenizerService(context)
 
     def teardown_method(self, _):
         shutil.rmtree(self.path)

helm/benchmark/adaptation/adapters/test_generation_adapter.py

@@ -11,29 +11,32 @@ from helm.benchmark.scenarios.scenario import (
     Input,
     Output,
 )
-from helm.benchmark.run_specs import get_scenario_spec1, get_adapter_spec1
+from helm.benchmark.run_specs.simple_run_specs import get_simple1_spec
 from helm.benchmark.adaptation.prompt import Prompt
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from .adapter_factory import AdapterFactory, ADAPT_GENERATION
-from .test_adapter import TestAdapter
+from helm.benchmark.adaptation.adapters.adapter_factory import AdapterFactory, ADAPT_GENERATION
+from helm.benchmark.adaptation.adapters.generation_adapter import GenerationAdapter
+from helm.benchmark.adaptation.adapters.test_adapter import TestAdapter
 
 
 class TestGenerationAdapter(TestAdapter):
     def test_adapt(self):
-        scenario = create_scenario(get_scenario_spec1())
-        adapter_spec = get_adapter_spec1()
+        run_spec = get_simple1_spec()
+        scenario = create_scenario(run_spec.scenario_spec)
+        adapter_spec = run_spec.adapter_spec
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
-        scenario_state = adapter.adapt(scenario.get_instances(output_path=""), parallelism=1)
+        instances = scenario.get_instances(output_path="")
+        request_states = adapter.adapt(instances, parallelism=1)
+        non_train_instances = [instance for instance in instances if instance.split != TRAIN_SPLIT]
 
         # Make sure we generated the right number of request_states:
         # For each trial, instance and reference (+ 1 for free-form generation).
-        num_instances = len(scenario_state.instances)
-        assert num_instances * adapter_spec.num_train_trials == len(scenario_state.request_states)
+        assert len(non_train_instances) * adapter_spec.num_train_trials == len(request_states)
 
     def test_construct_prompt(self):
         adapter_spec = AdapterSpec(
-            model="openai/davinci",
-            model_deployment="openai/davinci",
+            model="openai/gpt2",
+            model_deployment="huggingface/gpt2",
             method=ADAPT_GENERATION,
             input_prefix="",
             input_suffix="",
@@ -60,8 +63,8 @@ class TestGenerationAdapter(TestAdapter):
 
     def test_construct_prompt_with_truncation(self):
         adapter_spec = AdapterSpec(
-            model="openai/davinci",
-            model_deployment="openai/davinci",
+            model="openai/gpt2",
+            model_deployment="huggingface/gpt2",
             method=ADAPT_GENERATION,
             input_prefix="",
             output_prefix="",
@@ -83,11 +86,11 @@ class TestGenerationAdapter(TestAdapter):
 
         # Ensure that all the in-context examples were completely removed and we had to truncate the eval Instance input
         assert "train" not in prompt_text
-        assert prompt_text.count("eval") == 1948
+        assert prompt_text.count("eval") == 924
 
     def test_sample_examples_without_references(self):
         adapter_spec = AdapterSpec(
-            method=ADAPT_GENERATION, model="openai/ada", model_deployment="openai/ada", max_train_instances=1
+            method=ADAPT_GENERATION, model="openai/gpt2", model_deployment="huggingface/gpt2", max_train_instances=1
         )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         all_train_instances = [
@@ -101,7 +104,7 @@ class TestGenerationAdapter(TestAdapter):
 
     def test_sample_examples_open_ended_generation(self):
         adapter_spec = AdapterSpec(
-            method=ADAPT_GENERATION, model="openai/ada", model_deployment="openai/ada", max_train_instances=3
+            method=ADAPT_GENERATION, model="openai/gpt2", model_deployment="huggingface/gpt2", max_train_instances=3
         )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
 
@@ -117,7 +120,7 @@ class TestGenerationAdapter(TestAdapter):
 
     def test_sample_examples_open_ended_generation_stress(self):
         adapter_spec = AdapterSpec(
-            method=ADAPT_GENERATION, model="openai/ada", model_deployment="openai/ada", max_train_instances=5
+            method=ADAPT_GENERATION, model="openai/gpt2", model_deployment="huggingface/gpt2", max_train_instances=5
         )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
 
@@ -159,8 +162,8 @@ class TestGenerationAdapter(TestAdapter):
     def test_multiple_correct_reference(self):
         adapter_spec = AdapterSpec(
             method=ADAPT_GENERATION,
-            model="openai/ada",
-            model_deployment="openai/ada",
+            model="openai/gpt2",
+            model_deployment="huggingface/gpt2",
             max_train_instances=2,
             sample_train=False,
         )
@@ -194,7 +197,7 @@ class TestGenerationAdapter(TestAdapter):
             ],
             split=TEST_SPLIT,
         )
-        actual_instances = adapter.adapt(train_instances + [eval_instance], parallelism=1).request_states
+        actual_instances = adapter.adapt(train_instances + [eval_instance], parallelism=1)
         assert len(actual_instances) == 1
         assert actual_instances[0].request.prompt == (
             "Input: Second reference is correct\n"
@@ -208,8 +211,8 @@ class TestGenerationAdapter(TestAdapter):
     def test_multiple_correct_reference_multi_label(self):
         adapter_spec = AdapterSpec(
             method=ADAPT_GENERATION,
-            model="openai/ada",
-            model_deployment="openai/ada",
+            model="openai/gpt2",
+            model_deployment="huggingface/gpt2",
             max_train_instances=2,
             multi_label=True,
             sample_train=False,
@@ -244,7 +247,7 @@ class TestGenerationAdapter(TestAdapter):
             ],
             split=TEST_SPLIT,
         )
-        actual_instances = adapter.adapt(train_instances + [eval_instance], parallelism=1).request_states
+        actual_instances = adapter.adapt(train_instances + [eval_instance], parallelism=1)
         assert len(actual_instances) == 1
         assert actual_instances[0].request.prompt == (
             "Input: Second reference is correct\n"
@@ -254,3 +257,24 @@ class TestGenerationAdapter(TestAdapter):
             "Input: First reference is correct\n"
             "Output:"
         )
+
+    def test_construct_prompt_image_generation(self):
+        adapter_spec = AdapterSpec(
+            model_deployment="openai/dall-e-2",
+            method=ADAPT_GENERATION,
+            input_prefix="",
+            input_suffix="",
+            output_prefix="",
+            output_suffix="",
+            max_train_instances=0,
+            num_outputs=1,
+            max_tokens=0,
+        )
+        adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
+        assert isinstance(adapter, GenerationAdapter)
+
+        eval_instance = Instance(Input(text="a blue dog"), references=[])
+        prompt: Prompt = adapter.construct_prompt([], eval_instance, include_output=False, reference_index=None)
+
+        assert adapter.window_service.fits_within_context_window(prompt.text)
+        assert prompt.text == "a blue dog"

helm/benchmark/adaptation/adapters/test_language_modeling_adapter.py

@@ -1,35 +1,22 @@
 # mypy: check_untyped_defs = False
 from typing import List
-from helm.benchmark.window_services.gpt2_window_service import GPT2WindowService
 
 from helm.common.tokenization_request import TokenizationToken
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.common.request import Request
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from .adapter_factory import AdapterFactory, ADAPT_LANGUAGE_MODELING
-from .test_adapter import TestAdapter
+from helm.benchmark.adaptation.adapters.adapter_factory import AdapterFactory, ADAPT_LANGUAGE_MODELING
+from helm.benchmark.adaptation.adapters.test_adapter import TestAdapter
 from helm.benchmark.scenarios.scenario import TEST_SPLIT, Instance, Input, Reference
 
 
-class MockGPT2Window(GPT2WindowService):
-    """Utility for overriding properties of a GPT2WindowService for test purposes."""
-
-    def __init__(self, service, *, max_sequence_length):
-        super().__init__(service)
-        self._max_sequence_length = max_sequence_length
-
-    @property
-    def max_sequence_length(self) -> int:
-        return self._max_sequence_length
-
-
 class TestLanguageModelingAdapter(TestAdapter):
     def test_construct_language_modeling_prompt(self):
         adapter_spec = AdapterSpec(
             method=ADAPT_LANGUAGE_MODELING,
             input_prefix="",
-            model="openai/davinci",
-            model_deployment="openai/davinci",
+            model="openai/gpt2",
+            model_deployment="huggingface/gpt2",
             output_prefix="",
             max_tokens=0,
         )
@@ -52,8 +39,8 @@ class TestLanguageModelingAdapter(TestAdapter):
         adapter_spec = AdapterSpec(
             method=ADAPT_LANGUAGE_MODELING,
             input_prefix="",
-            model="openai/curie",
-            model_deployment="openai/curie",
+            model="openai/gpt2",
+            model_deployment="huggingface/gpt2",
             output_prefix="",
             max_tokens=0,
         )
@@ -69,10 +56,10 @@ class TestLanguageModelingAdapter(TestAdapter):
         )
 
         # Ensure the prompt is correct
-        assert prompt == "<|endoftext|>The" + " the" * 2047
+        assert prompt == "<|endoftext|>The" + " the" * 1023
 
         # Ensure the pred_tokens are correct
-        assert pred_tokens == [TokenizationToken(464)] + [TokenizationToken(262)] * 2047
+        assert pred_tokens == [TokenizationToken(464)] + [TokenizationToken(262)] * 1023
 
     def test_prompt_truncated(self):
         # Step 1. Test that the prompt is trucanted correctly when it is too long
@@ -100,7 +87,7 @@ class TestLanguageModelingAdapter(TestAdapter):
             split=TEST_SPLIT,
         )
         # Ensure the adapter returns the correct prompt
-        request_states: List[RequestState] = adapter.adapt([instance], parallelism=1).request_states
+        request_states: List[RequestState] = adapter.adapt([instance], parallelism=1)
         request: Request = request_states[0].request
         # The prompt should be "<|endoftext|>Excuse me, do you have the time?"
         assert request.prompt == "<|endoftext|>Excuse me, do you have the time?"
@@ -112,7 +99,7 @@ class TestLanguageModelingAdapter(TestAdapter):
             references=[reference],
             split=TEST_SPLIT,
         )
-        request_states_long: List[RequestState] = adapter.adapt([instance_long], parallelism=1).request_states
+        request_states_long: List[RequestState] = adapter.adapt([instance_long], parallelism=1)
         request_long: Request = request_states_long[0].request
         # Count the number of tokens of the prompt
         num_tokens = len(adapter.window_service.encode(request_long.prompt).token_values)
@@ -130,7 +117,7 @@ class TestLanguageModelingAdapter(TestAdapter):
         adapter_2 = AdapterFactory.get_adapter(adapter_spec_2_, self.tokenizer_service)
 
         # Step 2.1. Check that if the prompt is not too long, it is not truncated
-        request_state_2: List[RequestState] = adapter_2.adapt([instance], parallelism=1).request_states
+        request_state_2: List[RequestState] = adapter_2.adapt([instance], parallelism=1)
         request_2: Request = request_state_2[0].request
         # The prompt should be unchanged
         assert request_2.prompt == "<|endoftext|>Excuse me, do you have the time?"
@@ -138,7 +125,7 @@ class TestLanguageModelingAdapter(TestAdapter):
 
         # Step 2.2. Check that if the prompt + max_tokens is too long, it is truncated
         # but that we keep the same number of tokens as in the previous test
-        request_states_long_2: List[RequestState] = adapter_2.adapt([instance_long], parallelism=1).request_states
+        request_states_long_2: List[RequestState] = adapter_2.adapt([instance_long], parallelism=1)
         request_long_2: Request = request_states_long_2[0].request
         # Count the number of tokens of the prompt
         num_tokens_2 = len(adapter_2.window_service.encode(request_long_2.prompt).token_values)
@@ -152,19 +139,20 @@ class TestLanguageModelingAdapter(TestAdapter):
         adapter_spec = AdapterSpec(
             method=ADAPT_LANGUAGE_MODELING,
             input_prefix="",
-            model="openai/code-davinci-002",
-            model_deployment="openai/code-davinci-002",
+            model="openai/gpt2",
+            model_deployment="huggingface/gpt2",
             output_prefix="",
             max_tokens=0,
         )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         # Monkey patch the window service to have really short max sequences.
-        adapter.window_service = MockGPT2Window(self.tokenizer_service, max_sequence_length=max_sequence_length)
+        adapter.window_service._max_sequence_length = max_sequence_length
+        adapter.window_service._max_request_length = max_sequence_length + 1
         input_text = Input(text=" ".join(str(i) for i in range(input_tokens)))
         instance = Instance(input=input_text, references=[], split=TEST_SPLIT)
 
         # Generate the requests
-        request_states: List[RequestState] = adapter.adapt([instance], parallelism=1).request_states
+        request_states: List[RequestState] = adapter.adapt([instance], parallelism=1)
         # A smaller window service creates more requests
         assert len(request_states) == 3
         assert request_states[0].request.prompt == "<|endoftext|>0 1 2 3 4 5 6 7 8 9"