crfm-helm 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/benchmark/adaptation/adapters/test_multiple_choice_joint_adapter.py

@@ -1,10 +1,23 @@
 # mypy: check_untyped_defs = False
+from typing import List, Set
 from helm.benchmark.scenarios.scenario import TEST_SPLIT, TRAIN_SPLIT, Instance, Input, Output, Reference, CORRECT_TAG
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from .adapter_factory import AdapterFactory, ADAPT_MULTIPLE_CHOICE_JOINT
 from .test_adapter import TestAdapter
 
 
+def _make_instance(
+    text: str, reference_texts: List[str], correct_references: Set[int], is_eval: bool = False
+) -> Instance:
+    references = []
+    for i, reference_text in enumerate(reference_texts):
+        tags = [CORRECT_TAG] if i in correct_references else []
+        references.append(Reference(Output(text=reference_text), tags=tags))
+
+    split = TEST_SPLIT if is_eval else TRAIN_SPLIT
+    return Instance(Input(text=text), references=references, split=split)
+
+
 class TestMultipleChoiceJointAdapter(TestAdapter):
     def test_sample_examples(self):
         adapter_spec = AdapterSpec(
@@ -53,6 +66,47 @@ class TestMultipleChoiceJointAdapter(TestAdapter):
         examples = adapter.sample_examples(all_train_instances, seed=0)
         assert len(examples) == 3
 
+    def test_sample_examples_unique_labels(self):
+        """This is a demonstration of behavior reported in issue #2224."""
+        adapter_spec = AdapterSpec(
+            method=ADAPT_MULTIPLE_CHOICE_JOINT, model="openai/ada", model_deployment="openai/ada", max_train_instances=3
+        )
+        adapter = AdapterFactory.get_adapter(adapter_spec, self.tokenizer_service)
+        all_train_instances = [
+            # Three with 0 being correct.
+            _make_instance("one", ["0", "1"], correct_references={0}),
+            _make_instance("two", ["2", "3"], correct_references={0}),
+            _make_instance("three", ["4", "5"], correct_references={0}),
+            # Two with 1 being correct.
+            _make_instance("four", ["6", "7"], correct_references={1}),
+            _make_instance("five", ["8", "9"], correct_references={1}),
+        ]
+        eval_instance = _make_instance("eval", ["10", "11"], correct_references={1}, is_eval=True)
+        request_states = adapter.adapt(all_train_instances + [eval_instance], parallelism=1)
+        assert len(request_states) == 1
+        # In every case, we are showing that model that Output should be "A".
+        assert request_states[0].request.prompt == (
+            "Input: three\n"
+            "A. 4\n"
+            "B. 5\n"
+            "Output: A\n"
+            "\n"
+            "Input: two\n"
+            "A. 2\n"
+            "B. 3\n"
+            "Output: A\n"
+            "\n"
+            "Input: one\n"
+            "A. 0\n"
+            "B. 1\n"
+            "Output: A\n"
+            "\n"
+            "Input: eval\n"
+            "A. 10\n"
+            "B. 11\n"
+            "Output:"
+        )
+
     def test_multiple_correct_reference(self):
         adapter_spec = AdapterSpec(
             method=ADAPT_MULTIPLE_CHOICE_JOINT,
@@ -91,9 +145,9 @@ class TestMultipleChoiceJointAdapter(TestAdapter):
             ],
             split=TEST_SPLIT,
         )
-        actual_instances = adapter.adapt(train_instances + [eval_instance], parallelism=1).request_states
-        assert len(actual_instances) == 1
-        assert actual_instances[0].request.prompt == (
+        request_states = adapter.adapt(train_instances + [eval_instance], parallelism=1)
+        assert len(request_states) == 1
+        assert request_states[0].request.prompt == (
             "Input: Second reference is correct\n"
             "A. First\n"
             "B. Second\n"
@@ -150,9 +204,9 @@ class TestMultipleChoiceJointAdapter(TestAdapter):
             ],
             split=TEST_SPLIT,
         )
-        actual_instances = adapter.adapt(train_instances + [eval_instance], parallelism=1).request_states
-        assert len(actual_instances) == 1
-        assert actual_instances[0].request.prompt == (
+        request_states = adapter.adapt(train_instances + [eval_instance], parallelism=1)
+        assert len(request_states) == 1
+        assert request_states[0].request.prompt == (
             "Input: Second reference is correct\n"
             "A. First\n"
             "B. Second\n"

helm/benchmark/adaptation/common_adapter_specs.py

@@ -0,0 +1,376 @@
+from typing import List, Optional
+
+from helm.benchmark.adaptation.adapter_spec import (
+    ADAPT_GENERATION,
+    ADAPT_LANGUAGE_MODELING,
+    ADAPT_MULTIPLE_CHOICE_JOINT,
+    ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED,
+    ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL,
+    ADAPT_RANKING_BINARY,
+    AdapterSpec,
+)
+
+
+def format_instructions(instructions: str) -> str:
+    if len(instructions) > 0:
+        instructions += "\n"
+    return instructions
+
+
+def get_multiple_choice_joint_adapter_spec(
+    instructions: str,
+    input_noun: Optional[str],
+    output_noun: str,
+    num_outputs: int = 5,
+    max_train_instances: int = 5,
+    max_tokens: int = 5,
+    sample_train: bool = True,
+    **kwargs,
+) -> AdapterSpec:
+    """
+    [instructions]
+
+    [input_noun]: [input]
+    [reference_1]
+    ...
+    [reference_k]
+    [output_noun]: [output]
+
+    [input_noun]: [input]
+    [reference_1]
+    ...
+    [reference_k]
+    [output_noun]:
+    """
+
+    return AdapterSpec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions=format_instructions(instructions),
+        input_prefix=f"{input_noun}: " if input_noun is not None else "",
+        input_suffix="\n" if input_noun is not None else "",
+        output_prefix=f"{output_noun}: ",
+        output_suffix="\n",
+        max_train_instances=max_train_instances,
+        num_outputs=num_outputs,
+        max_tokens=max_tokens,
+        temperature=0.0,
+        stop_sequences=["\n"],
+        sample_train=sample_train,
+        **kwargs,
+    )
+
+
+def get_multiple_choice_separate_adapter_spec(method: str, empty_input: bool = False) -> AdapterSpec:
+    """
+    [input] [reference_i]
+    or
+    [reference_i]
+    """
+    assert method in {ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL, ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED}
+
+    return AdapterSpec(
+        method=method,
+        instructions="",
+        input_prefix="",
+        input_suffix="",
+        output_prefix=" " if not empty_input else "",
+        output_suffix="",
+        # Separate is basically language modeling, so can't easily use in-context examples
+        max_train_instances=0,
+        num_outputs=1,
+        max_tokens=0,
+        temperature=0.0,
+    )
+
+
+def get_multiple_choice_adapter_spec(
+    method: str,
+    instructions: str,
+    input_noun: Optional[str],
+    output_noun: str,
+    max_train_instances: int = 5,
+    num_outputs: int = 5,
+    max_tokens: int = 1,
+    empty_input: bool = False,
+    sample_train: bool = True,
+    **kwargs,
+):
+    """
+    Toggle between joint and separate adapters.
+    """
+    if method == ADAPT_MULTIPLE_CHOICE_JOINT:
+        return get_multiple_choice_joint_adapter_spec(
+            instructions,
+            input_noun,
+            output_noun,
+            max_train_instances=max_train_instances,
+            num_outputs=num_outputs,
+            max_tokens=max_tokens,
+            sample_train=sample_train,
+            **kwargs,
+        )
+    elif method in {ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL, ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED}:
+        return get_multiple_choice_separate_adapter_spec(method, empty_input)
+    else:
+        raise ValueError(f"Invalid adaptation method: {method}")
+
+
+def get_ranking_binary_adapter_spec(
+    instructions: str = "",
+    document_noun: str = "Passage",
+    query_noun: str = "Query",
+    output_prefix: str = "Does the passage answer the query?",
+    output_noun: str = "Answer",
+    max_train_instances: int = 4,
+    num_outputs: int = 1,
+    num_train_trials: int = 1,
+    temperature: float = 0.0,
+    max_tokens: int = 5,
+    **kwargs,
+) -> AdapterSpec:
+    """
+    [instructions]
+
+    [object_noun]: [object]
+    [query_noun]: [query]
+    [prompt_noun]: [prompt_content]
+    [output_noun]: [output]
+
+    ...
+
+    [object_noun]: [object]
+    [query_noun]: [query]
+    [prompt_noun]: [prompt_content]
+    [output_noun]: [output]
+
+    [object_noun]: [object]
+    [query_noun]: [query]
+    [prompt_noun]: [prompt_content]
+    [output_noun]: [output]
+    """
+    msg = (
+        "There must be an even number of in-context examples to ensure that"
+        "an equal number of positive and negative examples are included."
+    )
+    assert max_train_instances % 2 == 0, msg
+    max_train_instances = int(max_train_instances / 2)
+
+    return AdapterSpec(
+        method=ADAPT_RANKING_BINARY,
+        instructions=format_instructions(instructions),
+        input_prefix=f"{query_noun}: ",
+        input_suffix="\n",
+        reference_prefix=f"{document_noun}: ",
+        reference_suffix="\n",
+        output_prefix=f"{output_prefix}\n{output_noun}: ",
+        max_train_instances=max_train_instances,
+        num_outputs=num_outputs,
+        num_train_trials=num_train_trials,
+        temperature=temperature,
+        max_tokens=max_tokens,
+        **kwargs,
+    )
+
+
+def get_completion_adapter_spec(
+    instructions: str = "",
+    input_prefix: str = "",
+    output_prefix: str = "",
+    output_suffix: str = "",
+    max_train_instances: int = 0,
+    temperature: float = 0.0,
+    num_outputs: int = 1,
+    max_tokens: int = 100,
+    stop_sequences: Optional[List] = None,  # default value of `stop_sequences` is no stop sequence,
+    **kwargs,
+) -> AdapterSpec:
+    """
+    [input][output_prefix][output][output_suffix]
+
+    [input][output_prefix]
+    """
+    if stop_sequences is None:
+        stop_sequences = []
+
+    return AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions=format_instructions(instructions),
+        input_prefix=input_prefix,
+        input_suffix="",
+        output_prefix=output_prefix,
+        output_suffix=output_suffix,
+        max_train_instances=max_train_instances,
+        temperature=temperature,
+        num_outputs=num_outputs,
+        max_tokens=max_tokens,
+        stop_sequences=stop_sequences,
+        **kwargs,
+    )
+
+
+def get_generation_adapter_spec(
+    instructions: str = "",
+    input_noun: Optional[str] = None,
+    newline_after_input_noun: bool = False,
+    output_noun: Optional[str] = None,
+    newline_after_output_noun: bool = False,
+    max_train_instances: int = 5,
+    num_outputs: int = 1,
+    max_tokens: int = 5,
+    stop_sequences: Optional[List] = None,  # default value of `stop_sequences` is ["\n"]
+    temperature: float = 0.0,
+    multi_label: bool = False,
+) -> AdapterSpec:
+    """
+    [instructions]
+
+    [input_noun]: [input]
+    [output_noun]: [output]
+
+    [input_noun]: [input]
+    [output_noun]:
+    """
+
+    def format_prefix(noun: Optional[str], append_new_line: bool) -> str:
+        """
+        When `append_new_line` is False:
+            [input_noun]: [input]
+
+        When `append_new_line` is True:
+            [input_noun]:
+            [input]
+        """
+        prefix: str = f"{noun}:" if noun is not None else ""
+        if len(prefix) > 0:
+            prefix += "\n" if append_new_line else " "
+        return prefix
+
+    if stop_sequences is None:
+        stop_sequences = ["\n"]
+
+    return AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions=format_instructions(instructions),
+        input_prefix=format_prefix(input_noun, append_new_line=newline_after_input_noun),
+        input_suffix="\n",
+        output_prefix=format_prefix(output_noun, append_new_line=newline_after_output_noun),
+        output_suffix="\n",
+        max_train_instances=max_train_instances,
+        num_outputs=num_outputs,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        stop_sequences=stop_sequences,
+        multi_label=multi_label,
+    )
+
+
+def get_instruct_adapter_spec(
+    num_outputs: int = 1,
+    max_tokens: int = 512,
+    temperature: float = 0.7,
+) -> AdapterSpec:
+    """
+    Zero-shot instruction-following.
+    """
+    return AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions="",
+        input_prefix="",
+        input_suffix="\n",
+        output_prefix="",
+        output_suffix="",
+        max_train_instances=0,
+        num_outputs=num_outputs,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        stop_sequences=[],
+    )
+
+
+def get_few_shot_instruct_adapter_spec(
+    num_outputs: int = 1,
+    max_tokens: int = 512,
+    temperature: float = 0.7,
+    max_train_instances: int = 0,
+) -> AdapterSpec:
+    """
+    Few-shot instruction-following.
+    """
+    return AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions="",
+        input_prefix="",
+        input_suffix="\n",
+        output_prefix="",
+        output_suffix="",
+        max_train_instances=max_train_instances,
+        num_outputs=num_outputs,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        stop_sequences=[],
+    )
+
+
+def get_language_modeling_adapter_spec() -> AdapterSpec:
+    """
+    Used for language modeling.
+    """
+    return AdapterSpec(
+        method=ADAPT_LANGUAGE_MODELING,
+        instructions="",
+        input_prefix="",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        max_train_instances=0,
+        num_outputs=1,
+        max_tokens=0,
+        temperature=0.0,
+    )
+
+
+def get_summarization_adapter_spec(num_sents: Optional[int], max_train_instances: int = 5, **kwargs) -> AdapterSpec:
+    """
+    Used for summarization.
+    """
+
+    if num_sents == 1:
+        out_pref = "Summarize the above article in 1 sentence.\n"
+    elif num_sents is None:
+        out_pref = "Summarize the above article.\n"
+    else:
+        out_pref = f"Summarize the above article in {num_sents} sentences.\n"
+
+    return AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions="",
+        input_prefix="###\nArticle: ",
+        input_suffix="\n\n",
+        output_prefix=out_pref,
+        output_suffix="\n",
+        max_train_instances=max_train_instances,
+        num_outputs=1,
+        stop_sequences=["###"],  # Separator between few-shot instances.
+        **kwargs,
+    )
+
+
+def get_machine_translation_adapter_spec(
+    source_language, target_language, max_train_instances, **kwargs
+) -> AdapterSpec:
+    """
+    Used for machine translation.
+    """
+    return AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions=f"Translate the following sentences from {source_language} to {target_language}.",
+        input_prefix=f"{source_language}: ",
+        input_suffix="\n",
+        output_prefix=f"{target_language}: ",
+        output_suffix="\n",
+        max_train_instances=max_train_instances,
+        num_outputs=1,
+        stop_sequences=["\n\n"],
+        temperature=0.0,
+        **kwargs,
+    )

helm/benchmark/adaptation/request_state.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Optional, Dict, List
+from typing import Optional, Dict, List, Any
 
 from helm.benchmark.scenarios.scenario import Instance
 from helm.common.general import indent_lines, format_text_lines, serialize
@@ -45,6 +45,11 @@ class RequestState:
     num_conditioning_tokens: int = 0
     """The number of initial tokens that will be ignored when computing language modeling metrics"""
 
+    annotations: Optional[Dict[str, Any]] = None
+    """Output of some post-processing step that is needed for the metric to understand the request
+    Should match the annotator's name to an Annotation (usually a list of dictionaries for each completion)
+    Example: parsing, rendering an image based on the text completion, etc."""
+
     def __post_init__(self):
         if self.request_mode:
             assert self.request_mode in ["original", "calibration"], f"Invalid request_mode: {self.request_mode}"

helm/benchmark/adaptation/scenario_state.py

@@ -3,8 +3,9 @@ from dataclasses import dataclass
 from typing import List, Dict, Tuple, Optional
 
 from helm.benchmark.scenarios.scenario import Instance
-from .adapter_spec import AdapterSpec
-from .request_state import RequestState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import AnnotatorSpec
 
 
 @dataclass
@@ -21,6 +22,9 @@ class ScenarioState:
     # List of `RequestState`s that were produced by adaptation (and execution)
     request_states: List[RequestState]
 
+    # Annotations to use for this run spec
+    annotator_specs: Optional[List[AnnotatorSpec]] = None
+
     def __post_init__(self):
         # Create derived indices based on `request_states` so it's easier for
         # the `Metric` later to access them.  Two things are produced:

helm/benchmark/annotation/annotator.py

@@ -0,0 +1,43 @@
+from typing import Dict, List, Any
+from abc import abstractmethod, ABC
+from dataclasses import dataclass
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.common.object_spec import ObjectSpec
+
+
+class Annotator(ABC):
+    """Annotator is an abstract class for annotating a request state. Annotators are used to add additional
+    information to a request state that is needed for a metric to understand the request. This could be
+    parsing, rendering an image based on the text completion, etc."""
+
+    name: str
+    """Name of the annotator. Should be filled in by the subclass."""
+
+    @abstractmethod
+    def annotate(self, request_state: RequestState) -> Any:
+        """Fills the annotations field of the request state with additional information
+        that are implementation specific."""
+        pass
+
+
+@dataclass(frozen=True)
+class AnnotatorSpec(ObjectSpec):
+    """Specifies how to create an `Annotator`.
+    The user should only specify the class name.
+    The arguments will be filled in by the `AnnotatorFactory`.
+    """
+
+    pass
+
+
+class DummyAnnotator(Annotator):
+    """A dummy annotator that does nothing."""
+
+    name = "dummy"
+
+    def annotate(self, request_state: RequestState) -> List[Dict[str, Any]]:
+        if request_state.result is None:
+            raise ValueError("Annotation requires a result")
+        annotation_values: List[str] = [completion.text.upper() for completion in request_state.result.completions]
+        return [{"all_caps": value} for value in annotation_values]

helm/benchmark/annotation/annotator_factory.py

@@ -0,0 +1,61 @@
+import os
+from typing import Any, Dict, Mapping, Optional
+
+from helm.common.credentials_utils import provide_api_key
+from helm.common.cache_backend_config import CacheBackendConfig, CacheConfig
+from helm.common.hierarchical_logger import hlog
+from helm.common.object_spec import create_object, inject_object_spec_args
+from helm.benchmark.annotation.annotator import Annotator, AnnotatorSpec
+
+
+class AnnotatorFactory:
+    """Factory for creating annotators."""
+
+    def __init__(
+        self, credentials: Mapping[str, Any], file_storage_path: str, cache_backend_config: CacheBackendConfig
+    ):
+        self.credentials = credentials
+        self.file_storage_path = file_storage_path
+        self.cache_backend_config = cache_backend_config
+        hlog(f"AnnotatorFactory: file_storage_path = {file_storage_path}")
+        hlog(f"AnnotatorFactory: cache_backend_config = {cache_backend_config}")
+
+        # Cache for annotators
+        # This is used to prevent duplicate creation of annotators
+        # It is especially important as annotation is a multi-threaded
+        # process and creating a new annotator for each request can cause
+        # race conditions.
+        self.annotators: Dict[str, Annotator] = {}
+
+    def get_annotator(self, annotator_spec: AnnotatorSpec) -> Annotator:
+        """Return a annotator based on the name."""
+        # First try to find the annotator in the cache
+        assert annotator_spec.args is None or annotator_spec.args == {}
+        annotator_name: str = annotator_spec.class_name.split(".")[-1].lower().replace("annotator", "")
+        annotator: Optional[Annotator] = self.annotators.get(annotator_name)
+        if annotator is not None:
+            return annotator
+
+        # Otherwise, create the client
+        cache_config: CacheConfig = self.cache_backend_config.get_cache_config(annotator_name)
+        annotator_spec = inject_object_spec_args(
+            annotator_spec,
+            constant_bindings={
+                "cache_config": cache_config,
+            },
+            provider_bindings={
+                "api_key": lambda: provide_api_key(self.credentials, annotator_name),
+                "file_storage_path": lambda: self._get_file_storage_path(annotator_name),
+            },
+        )
+        annotator = create_object(annotator_spec)
+
+        # Cache the client
+        self.annotators[annotator_name] = annotator
+
+        return annotator
+
+    def _get_file_storage_path(self, annotator_name: str) -> str:
+        # Returns the path to use for a local file cache for the given annotator
+        local_file_cache_path: str = os.path.join(self.file_storage_path, "output", annotator_name)
+        return local_file_cache_path