crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/adaptation/adapters/test_generation_adapter.py

@@ -11,28 +11,32 @@ from helm.benchmark.scenarios.scenario import (
     Input,
     Output,
 )
-from helm.benchmark.run_specs import get_scenario_spec1, get_adapter_spec1
+from helm.benchmark.run_specs.simple_run_specs import get_simple1_spec
 from helm.benchmark.adaptation.prompt import Prompt
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from .adapter_factory import AdapterFactory, ADAPT_GENERATION
+from .generation_adapter import GenerationAdapter
 from .test_adapter import TestAdapter
 
 
 class TestGenerationAdapter(TestAdapter):
     def test_adapt(self):
-        scenario = create_scenario(get_scenario_spec1())
-        adapter_spec = get_adapter_spec1()
+        run_spec = get_simple1_spec()
+        scenario = create_scenario(run_spec.scenario_spec)
+        adapter_spec = run_spec.adapter_spec
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
-        scenario_state = adapter.adapt(scenario.get_instances(output_path=""), parallelism=1)
+        instances = scenario.get_instances(output_path="")
+        request_states = adapter.adapt(instances, parallelism=1)
+        non_train_instances = [instance for instance in instances if instance.split != TRAIN_SPLIT]
 
         # Make sure we generated the right number of request_states:
         # For each trial, instance and reference (+ 1 for free-form generation).
-        num_instances = len(scenario_state.instances)
-        assert num_instances * adapter_spec.num_train_trials == len(scenario_state.request_states)
+        assert len(non_train_instances) * adapter_spec.num_train_trials == len(request_states)
 
     def test_construct_prompt(self):
         adapter_spec = AdapterSpec(
             model="openai/davinci",
+            model_deployment="openai/davinci",
             method=ADAPT_GENERATION,
             input_prefix="",
             input_suffix="",
@@ -59,7 +63,12 @@ class TestGenerationAdapter(TestAdapter):
 
     def test_construct_prompt_with_truncation(self):
         adapter_spec = AdapterSpec(
-            model="openai/davinci", method=ADAPT_GENERATION, input_prefix="", output_prefix="", max_tokens=100
+            model="openai/davinci",
+            model_deployment="openai/davinci",
+            method=ADAPT_GENERATION,
+            input_prefix="",
+            output_prefix="",
+            max_tokens=100,
         )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         correct_reference = Reference(Output(text=""), tags=[CORRECT_TAG])
@@ -80,7 +89,9 @@ class TestGenerationAdapter(TestAdapter):
         assert prompt_text.count("eval") == 1948
 
     def test_sample_examples_without_references(self):
-        adapter_spec = AdapterSpec(method=ADAPT_GENERATION, model="openai/ada", max_train_instances=1)
+        adapter_spec = AdapterSpec(
+            method=ADAPT_GENERATION, model="openai/ada", model_deployment="openai/ada", max_train_instances=1
+        )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         all_train_instances = [
             Instance(Input(text="prompt1"), references=[]),
@@ -92,7 +103,9 @@ class TestGenerationAdapter(TestAdapter):
         assert len(examples) == 1
 
     def test_sample_examples_open_ended_generation(self):
-        adapter_spec = AdapterSpec(method=ADAPT_GENERATION, model="openai/ada", max_train_instances=3)
+        adapter_spec = AdapterSpec(
+            method=ADAPT_GENERATION, model="openai/ada", model_deployment="openai/ada", max_train_instances=3
+        )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
 
         all_train_instances: List[Instance] = [
@@ -106,7 +119,9 @@ class TestGenerationAdapter(TestAdapter):
         assert seed0_examples != seed1_examples, "Examples should differ when changing the seed"
 
     def test_sample_examples_open_ended_generation_stress(self):
-        adapter_spec = AdapterSpec(method=ADAPT_GENERATION, model="openai/ada", max_train_instances=5)
+        adapter_spec = AdapterSpec(
+            method=ADAPT_GENERATION, model="openai/ada", model_deployment="openai/ada", max_train_instances=5
+        )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
 
         all_train_instances: List[Instance] = [
@@ -146,7 +161,11 @@ class TestGenerationAdapter(TestAdapter):
 
     def test_multiple_correct_reference(self):
         adapter_spec = AdapterSpec(
-            method=ADAPT_GENERATION, model="openai/ada", max_train_instances=2, sample_train=False
+            method=ADAPT_GENERATION,
+            model="openai/ada",
+            model_deployment="openai/ada",
+            max_train_instances=2,
+            sample_train=False,
         )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         train_instances = [
@@ -178,7 +197,7 @@ class TestGenerationAdapter(TestAdapter):
             ],
             split=TEST_SPLIT,
         )
-        actual_instances = adapter.adapt(train_instances + [eval_instance], parallelism=1).request_states
+        actual_instances = adapter.adapt(train_instances + [eval_instance], parallelism=1)
         assert len(actual_instances) == 1
         assert actual_instances[0].request.prompt == (
             "Input: Second reference is correct\n"
@@ -191,7 +210,12 @@ class TestGenerationAdapter(TestAdapter):
 
     def test_multiple_correct_reference_multi_label(self):
         adapter_spec = AdapterSpec(
-            method=ADAPT_GENERATION, model="openai/ada", max_train_instances=2, multi_label=True, sample_train=False
+            method=ADAPT_GENERATION,
+            model="openai/ada",
+            model_deployment="openai/ada",
+            max_train_instances=2,
+            multi_label=True,
+            sample_train=False,
         )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         train_instances = [
@@ -223,7 +247,7 @@ class TestGenerationAdapter(TestAdapter):
             ],
             split=TEST_SPLIT,
         )
-        actual_instances = adapter.adapt(train_instances + [eval_instance], parallelism=1).request_states
+        actual_instances = adapter.adapt(train_instances + [eval_instance], parallelism=1)
         assert len(actual_instances) == 1
         assert actual_instances[0].request.prompt == (
             "Input: Second reference is correct\n"
@@ -233,3 +257,24 @@ class TestGenerationAdapter(TestAdapter):
             "Input: First reference is correct\n"
             "Output:"
         )
+
+    def test_construct_prompt_image_generation(self):
+        adapter_spec = AdapterSpec(
+            model_deployment="openai/dall-e-2",
+            method=ADAPT_GENERATION,
+            input_prefix="",
+            input_suffix="",
+            output_prefix="",
+            output_suffix="",
+            max_train_instances=0,
+            num_outputs=1,
+            max_tokens=0,
+        )
+        adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
+        assert isinstance(adapter, GenerationAdapter)
+
+        eval_instance = Instance(Input(text="a blue dog"), references=[])
+        prompt: Prompt = adapter.construct_prompt([], eval_instance, include_output=False, reference_index=None)
+
+        assert adapter.window_service.fits_within_context_window(prompt.text)
+        assert prompt.text == "a blue dog"

helm/benchmark/adaptation/adapters/test_language_modeling_adapter.py

@@ -7,7 +7,7 @@ from helm.common.request import Request
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from .adapter_factory import AdapterFactory, ADAPT_LANGUAGE_MODELING
 from .test_adapter import TestAdapter
-from helm.benchmark.scenarios.scenario import Instance, Input, Reference
+from helm.benchmark.scenarios.scenario import TEST_SPLIT, Instance, Input, Reference
 
 
 class TestLanguageModelingAdapter(TestAdapter):
@@ -16,6 +16,7 @@ class TestLanguageModelingAdapter(TestAdapter):
             method=ADAPT_LANGUAGE_MODELING,
             input_prefix="",
             model="openai/davinci",
+            model_deployment="openai/davinci",
             output_prefix="",
             max_tokens=0,
         )
@@ -39,6 +40,7 @@ class TestLanguageModelingAdapter(TestAdapter):
             method=ADAPT_LANGUAGE_MODELING,
             input_prefix="",
             model="openai/curie",
+            model_deployment="openai/curie",
             output_prefix="",
             max_tokens=0,
         )
@@ -70,6 +72,7 @@ class TestLanguageModelingAdapter(TestAdapter):
             method=ADAPT_LANGUAGE_MODELING,
             input_prefix="",
             model="anthropic/claude-v1.3",
+            model_deployment="anthropic/claude-v1.3",
             output_prefix="",
             max_tokens=0,
         )
@@ -81,9 +84,10 @@ class TestLanguageModelingAdapter(TestAdapter):
         instance: Instance = Instance(
             input=input_text,
             references=[reference],
+            split=TEST_SPLIT,
         )
         # Ensure the adapter returns the correct prompt
-        request_states: List[RequestState] = adapter.adapt([instance], parallelism=1).request_states
+        request_states: List[RequestState] = adapter.adapt([instance], parallelism=1)
         request: Request = request_states[0].request
         # The prompt should be "<|endoftext|>Excuse me, do you have the time?"
         assert request.prompt == "<|endoftext|>Excuse me, do you have the time?"
@@ -93,8 +97,9 @@ class TestLanguageModelingAdapter(TestAdapter):
         instance_long: Instance = Instance(
             input=input_text_long,
             references=[reference],
+            split=TEST_SPLIT,
         )
-        request_states_long: List[RequestState] = adapter.adapt([instance_long], parallelism=1).request_states
+        request_states_long: List[RequestState] = adapter.adapt([instance_long], parallelism=1)
         request_long: Request = request_states_long[0].request
         # Count the number of tokens of the prompt
         num_tokens = len(adapter.window_service.encode(request_long.prompt).token_values)
@@ -105,13 +110,14 @@ class TestLanguageModelingAdapter(TestAdapter):
             method=ADAPT_LANGUAGE_MODELING,
             input_prefix="",
             model="anthropic/claude-v1.3",
+            model_deployment="anthropic/claude-v1.3",
             output_prefix="",
             max_tokens=2000,
         )
         adapter_2 = AdapterFactory.get_adapter(adapter_spec_2_, self.tokenizer_service)
 
         # Step 2.1. Check that if the prompt is not too long, it is not truncated
-        request_state_2: List[RequestState] = adapter_2.adapt([instance], parallelism=1).request_states
+        request_state_2: List[RequestState] = adapter_2.adapt([instance], parallelism=1)
         request_2: Request = request_state_2[0].request
         # The prompt should be unchanged
         assert request_2.prompt == "<|endoftext|>Excuse me, do you have the time?"
@@ -119,9 +125,38 @@ class TestLanguageModelingAdapter(TestAdapter):
 
         # Step 2.2. Check that if the prompt + max_tokens is too long, it is truncated
         # but that we keep the same number of tokens as in the previous test
-        request_states_long_2: List[RequestState] = adapter_2.adapt([instance_long], parallelism=1).request_states
+        request_states_long_2: List[RequestState] = adapter_2.adapt([instance_long], parallelism=1)
         request_long_2: Request = request_states_long_2[0].request
         # Count the number of tokens of the prompt
         num_tokens_2 = len(adapter_2.window_service.encode(request_long_2.prompt).token_values)
         assert num_tokens_2 == adapter.window_service.max_sequence_and_generated_tokens_length - 2000
         assert request_long_2.max_tokens == 2000
+
+    # TODO(#1969) Determine if this behavior is actually desirable.
+    def test_prompt_wrapping(self):
+        input_tokens = 25
+        max_sequence_length = 10
+        adapter_spec = AdapterSpec(
+            method=ADAPT_LANGUAGE_MODELING,
+            input_prefix="",
+            model="openai/code-davinci-002",
+            model_deployment="openai/code-davinci-002",
+            output_prefix="",
+            max_tokens=0,
+        )
+        adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
+        # Monkey patch the window service to have really short max sequences.
+        adapter.window_service._max_sequence_length = max_sequence_length
+        adapter.window_service._max_request_length = max_sequence_length + 1
+        input_text = Input(text=" ".join(str(i) for i in range(input_tokens)))
+        instance = Instance(input=input_text, references=[], split=TEST_SPLIT)
+
+        # Generate the requests
+        request_states: List[RequestState] = adapter.adapt([instance], parallelism=1)
+        # A smaller window service creates more requests
+        assert len(request_states) == 3
+        assert request_states[0].request.prompt == "<|endoftext|>0 1 2 3 4 5 6 7 8 9"
+        # Only the first prompt inclues the prefix_token
+        assert request_states[1].request.prompt == " 9 10 11 12 13 14 15 16 17 18 19"
+        # The last prompt includes as many conditioning_tokens as will fit
+        assert request_states[2].request.prompt == " 14 15 16 17 18 19 20 21 22 23 24"

helm/benchmark/adaptation/adapters/test_multiple_choice_joint_adapter.py

@@ -1,13 +1,28 @@
 # mypy: check_untyped_defs = False
+from typing import List, Set
 from helm.benchmark.scenarios.scenario import TEST_SPLIT, TRAIN_SPLIT, Instance, Input, Output, Reference, CORRECT_TAG
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from .adapter_factory import AdapterFactory, ADAPT_MULTIPLE_CHOICE_JOINT
 from .test_adapter import TestAdapter
 
 
+def _make_instance(
+    text: str, reference_texts: List[str], correct_references: Set[int], is_eval: bool = False
+) -> Instance:
+    references = []
+    for i, reference_text in enumerate(reference_texts):
+        tags = [CORRECT_TAG] if i in correct_references else []
+        references.append(Reference(Output(text=reference_text), tags=tags))
+
+    split = TEST_SPLIT if is_eval else TRAIN_SPLIT
+    return Instance(Input(text=text), references=references, split=split)
+
+
 class TestMultipleChoiceJointAdapter(TestAdapter):
     def test_sample_examples(self):
-        adapter_spec = AdapterSpec(method=ADAPT_MULTIPLE_CHOICE_JOINT, model="openai/ada", max_train_instances=4)
+        adapter_spec = AdapterSpec(
+            method=ADAPT_MULTIPLE_CHOICE_JOINT, model="openai/ada", model_deployment="openai/ada", max_train_instances=4
+        )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         all_train_instances = [
             Instance(Input(text="say no"), references=[Reference(Output(text="no"), tags=[CORRECT_TAG])]),
@@ -27,13 +42,20 @@ class TestMultipleChoiceJointAdapter(TestAdapter):
         assert examples[3].input.text == "say yes3"
 
     def test_sample_examples_no_train_instances(self):
-        adapter_spec = AdapterSpec(method=ADAPT_MULTIPLE_CHOICE_JOINT, model="openai/ada", max_train_instances=2)
+        adapter_spec = AdapterSpec(
+            method=ADAPT_MULTIPLE_CHOICE_JOINT, model="openai/ada", model_deployment="openai/ada", max_train_instances=2
+        )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         examples = adapter.sample_examples(all_train_instances=[], seed=0)
         assert len(examples) == 0
 
     def test_sample_examples_greater_max_train_instances(self):
-        adapter_spec = AdapterSpec(method=ADAPT_MULTIPLE_CHOICE_JOINT, model="openai/ada", max_train_instances=10)
+        adapter_spec = AdapterSpec(
+            method=ADAPT_MULTIPLE_CHOICE_JOINT,
+            model="openai/ada",
+            model_deployment="openai/ada",
+            max_train_instances=10,
+        )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         all_train_instances = [
             Instance(Input(text="say no"), references=[Reference(Output(text="no"), tags=[CORRECT_TAG])]),
@@ -44,9 +66,54 @@ class TestMultipleChoiceJointAdapter(TestAdapter):
         examples = adapter.sample_examples(all_train_instances, seed=0)
         assert len(examples) == 3
 
+    def test_sample_examples_unique_labels(self):
+        """This is a demonstration of behavior reported in issue #2224."""
+        adapter_spec = AdapterSpec(
+            method=ADAPT_MULTIPLE_CHOICE_JOINT, model="openai/ada", model_deployment="openai/ada", max_train_instances=3
+        )
+        adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
+        all_train_instances = [
+            # Three with 0 being correct.
+            _make_instance("one", ["0", "1"], correct_references={0}),
+            _make_instance("two", ["2", "3"], correct_references={0}),
+            _make_instance("three", ["4", "5"], correct_references={0}),
+            # Two with 1 being correct.
+            _make_instance("four", ["6", "7"], correct_references={1}),
+            _make_instance("five", ["8", "9"], correct_references={1}),
+        ]
+        eval_instance = _make_instance("eval", ["10", "11"], correct_references={1}, is_eval=True)
+        request_states = adapter.adapt(all_train_instances + [eval_instance], parallelism=1)
+        assert len(request_states) == 1
+        # In every case, we are showing that model that Output should be "A".
+        assert request_states[0].request.prompt == (
+            "Input: three\n"
+            "A. 4\n"
+            "B. 5\n"
+            "Output: A\n"
+            "\n"
+            "Input: two\n"
+            "A. 2\n"
+            "B. 3\n"
+            "Output: A\n"
+            "\n"
+            "Input: one\n"
+            "A. 0\n"
+            "B. 1\n"
+            "Output: A\n"
+            "\n"
+            "Input: eval\n"
+            "A. 10\n"
+            "B. 11\n"
+            "Output:"
+        )
+
     def test_multiple_correct_reference(self):
         adapter_spec = AdapterSpec(
-            method=ADAPT_MULTIPLE_CHOICE_JOINT, model="openai/ada", max_train_instances=10, sample_train=False
+            method=ADAPT_MULTIPLE_CHOICE_JOINT,
+            model="openai/ada",
+            model_deployment="openai/ada",
+            max_train_instances=10,
+            sample_train=False,
         )
         adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
         train_instances = [
@@ -78,9 +145,9 @@ class TestMultipleChoiceJointAdapter(TestAdapter):
             ],
             split=TEST_SPLIT,
         )
-        actual_instances = adapter.adapt(train_instances + [eval_instance], parallelism=1).request_states
-        assert len(actual_instances) == 1
-        assert actual_instances[0].request.prompt == (
+        request_states = adapter.adapt(train_instances + [eval_instance], parallelism=1)
+        assert len(request_states) == 1
+        assert request_states[0].request.prompt == (
             "Input: Second reference is correct\n"
             "A. First\n"
             "B. Second\n"
@@ -102,6 +169,7 @@ class TestMultipleChoiceJointAdapter(TestAdapter):
         adapter_spec = AdapterSpec(
             method=ADAPT_MULTIPLE_CHOICE_JOINT,
             model="openai/ada",
+            model_deployment="openai/ada",
             max_train_instances=10,
             multi_label=True,
             sample_train=False,
@@ -136,9 +204,9 @@ class TestMultipleChoiceJointAdapter(TestAdapter):
             ],
             split=TEST_SPLIT,
         )
-        actual_instances = adapter.adapt(train_instances + [eval_instance], parallelism=1).request_states
-        assert len(actual_instances) == 1
-        assert actual_instances[0].request.prompt == (
+        request_states = adapter.adapt(train_instances + [eval_instance], parallelism=1)
+        assert len(request_states) == 1
+        assert request_states[0].request.prompt == (
             "Input: Second reference is correct\n"
             "A. First\n"
             "B. Second\n"