crfm-helm 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/adaptation/common_adapter_specs.py

@@ -0,0 +1,376 @@
+from typing import List, Optional
+
+from helm.benchmark.adaptation.adapter_spec import (
+    ADAPT_GENERATION,
+    ADAPT_LANGUAGE_MODELING,
+    ADAPT_MULTIPLE_CHOICE_JOINT,
+    ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED,
+    ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL,
+    ADAPT_RANKING_BINARY,
+    AdapterSpec,
+)
+
+
+def format_instructions(instructions: str) -> str:
+    if len(instructions) > 0:
+        instructions += "\n"
+    return instructions
+
+
+def get_multiple_choice_joint_adapter_spec(
+    instructions: str,
+    input_noun: Optional[str],
+    output_noun: str,
+    num_outputs: int = 5,
+    max_train_instances: int = 5,
+    max_tokens: int = 5,
+    sample_train: bool = True,
+    **kwargs,
+) -> AdapterSpec:
+    """
+    [instructions]
+
+    [input_noun]: [input]
+    [reference_1]
+    ...
+    [reference_k]
+    [output_noun]: [output]
+
+    [input_noun]: [input]
+    [reference_1]
+    ...
+    [reference_k]
+    [output_noun]:
+    """
+
+    return AdapterSpec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions=format_instructions(instructions),
+        input_prefix=f"{input_noun}: " if input_noun is not None else "",
+        input_suffix="\n" if input_noun is not None else "",
+        output_prefix=f"{output_noun}: ",
+        output_suffix="\n",
+        max_train_instances=max_train_instances,
+        num_outputs=num_outputs,
+        max_tokens=max_tokens,
+        temperature=0.0,
+        stop_sequences=["\n"],
+        sample_train=sample_train,
+        **kwargs,
+    )
+
+
+def get_multiple_choice_separate_adapter_spec(method: str, empty_input: bool = False) -> AdapterSpec:
+    """
+    [input] [reference_i]
+    or
+    [reference_i]
+    """
+    assert method in {ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL, ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED}
+
+    return AdapterSpec(
+        method=method,
+        instructions="",
+        input_prefix="",
+        input_suffix="",
+        output_prefix=" " if not empty_input else "",
+        output_suffix="",
+        # Separate is basically language modeling, so can't easily use in-context examples
+        max_train_instances=0,
+        num_outputs=1,
+        max_tokens=0,
+        temperature=0.0,
+    )
+
+
+def get_multiple_choice_adapter_spec(
+    method: str,
+    instructions: str,
+    input_noun: Optional[str],
+    output_noun: str,
+    max_train_instances: int = 5,
+    num_outputs: int = 5,
+    max_tokens: int = 1,
+    empty_input: bool = False,
+    sample_train: bool = True,
+    **kwargs,
+):
+    """
+    Toggle between joint and separate adapters.
+    """
+    if method == ADAPT_MULTIPLE_CHOICE_JOINT:
+        return get_multiple_choice_joint_adapter_spec(
+            instructions,
+            input_noun,
+            output_noun,
+            max_train_instances=max_train_instances,
+            num_outputs=num_outputs,
+            max_tokens=max_tokens,
+            sample_train=sample_train,
+            **kwargs,
+        )
+    elif method in {ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL, ADAPT_MULTIPLE_CHOICE_SEPARATE_CALIBRATED}:
+        return get_multiple_choice_separate_adapter_spec(method, empty_input)
+    else:
+        raise ValueError(f"Invalid adaptation method: {method}")
+
+
+def get_ranking_binary_adapter_spec(
+    instructions: str = "",
+    document_noun: str = "Passage",
+    query_noun: str = "Query",
+    output_prefix: str = "Does the passage answer the query?",
+    output_noun: str = "Answer",
+    max_train_instances: int = 4,
+    num_outputs: int = 1,
+    num_train_trials: int = 1,
+    temperature: float = 0.0,
+    max_tokens: int = 5,
+    **kwargs,
+) -> AdapterSpec:
+    """
+    [instructions]
+
+    [object_noun]: [object]
+    [query_noun]: [query]
+    [prompt_noun]: [prompt_content]
+    [output_noun]: [output]
+
+    ...
+
+    [object_noun]: [object]
+    [query_noun]: [query]
+    [prompt_noun]: [prompt_content]
+    [output_noun]: [output]
+
+    [object_noun]: [object]
+    [query_noun]: [query]
+    [prompt_noun]: [prompt_content]
+    [output_noun]: [output]
+    """
+    msg = (
+        "There must be an even number of in-context examples to ensure that"
+        "an equal number of positive and negative examples are included."
+    )
+    assert max_train_instances % 2 == 0, msg
+    max_train_instances = int(max_train_instances / 2)
+
+    return AdapterSpec(
+        method=ADAPT_RANKING_BINARY,
+        instructions=format_instructions(instructions),
+        input_prefix=f"{query_noun}: ",
+        input_suffix="\n",
+        reference_prefix=f"{document_noun}: ",
+        reference_suffix="\n",
+        output_prefix=f"{output_prefix}\n{output_noun}: ",
+        max_train_instances=max_train_instances,
+        num_outputs=num_outputs,
+        num_train_trials=num_train_trials,
+        temperature=temperature,
+        max_tokens=max_tokens,
+        **kwargs,
+    )
+
+
+def get_completion_adapter_spec(
+    instructions: str = "",
+    input_prefix: str = "",
+    output_prefix: str = "",
+    output_suffix: str = "",
+    max_train_instances: int = 0,
+    temperature: float = 0.0,
+    num_outputs: int = 1,
+    max_tokens: int = 100,
+    stop_sequences: Optional[List] = None,  # default value of `stop_sequences` is no stop sequence,
+    **kwargs,
+) -> AdapterSpec:
+    """
+    [input][output_prefix][output][output_suffix]
+
+    [input][output_prefix]
+    """
+    if stop_sequences is None:
+        stop_sequences = []
+
+    return AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions=format_instructions(instructions),
+        input_prefix=input_prefix,
+        input_suffix="",
+        output_prefix=output_prefix,
+        output_suffix=output_suffix,
+        max_train_instances=max_train_instances,
+        temperature=temperature,
+        num_outputs=num_outputs,
+        max_tokens=max_tokens,
+        stop_sequences=stop_sequences,
+        **kwargs,
+    )
+
+
+def get_generation_adapter_spec(
+    instructions: str = "",
+    input_noun: Optional[str] = None,
+    newline_after_input_noun: bool = False,
+    output_noun: Optional[str] = None,
+    newline_after_output_noun: bool = False,
+    max_train_instances: int = 5,
+    num_outputs: int = 1,
+    max_tokens: int = 5,
+    stop_sequences: Optional[List] = None,  # default value of `stop_sequences` is ["\n"]
+    temperature: float = 0.0,
+    multi_label: bool = False,
+) -> AdapterSpec:
+    """
+    [instructions]
+
+    [input_noun]: [input]
+    [output_noun]: [output]
+
+    [input_noun]: [input]
+    [output_noun]:
+    """
+
+    def format_prefix(noun: Optional[str], append_new_line: bool) -> str:
+        """
+        When `append_new_line` is False:
+            [input_noun]: [input]
+
+        When `append_new_line` is True:
+            [input_noun]:
+            [input]
+        """
+        prefix: str = f"{noun}:" if noun is not None else ""
+        if len(prefix) > 0:
+            prefix += "\n" if append_new_line else " "
+        return prefix
+
+    if stop_sequences is None:
+        stop_sequences = ["\n"]
+
+    return AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions=format_instructions(instructions),
+        input_prefix=format_prefix(input_noun, append_new_line=newline_after_input_noun),
+        input_suffix="\n",
+        output_prefix=format_prefix(output_noun, append_new_line=newline_after_output_noun),
+        output_suffix="\n",
+        max_train_instances=max_train_instances,
+        num_outputs=num_outputs,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        stop_sequences=stop_sequences,
+        multi_label=multi_label,
+    )
+
+
+def get_instruct_adapter_spec(
+    num_outputs: int = 1,
+    max_tokens: int = 512,
+    temperature: float = 0.7,
+) -> AdapterSpec:
+    """
+    Zero-shot instruction-following.
+    """
+    return AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions="",
+        input_prefix="",
+        input_suffix="\n",
+        output_prefix="",
+        output_suffix="",
+        max_train_instances=0,
+        num_outputs=num_outputs,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        stop_sequences=[],
+    )
+
+
+def get_few_shot_instruct_adapter_spec(
+    num_outputs: int = 1,
+    max_tokens: int = 512,
+    temperature: float = 0.7,
+    max_train_instances: int = 0,
+) -> AdapterSpec:
+    """
+    Few-shot instruction-following.
+    """
+    return AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions="",
+        input_prefix="",
+        input_suffix="\n",
+        output_prefix="",
+        output_suffix="",
+        max_train_instances=max_train_instances,
+        num_outputs=num_outputs,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        stop_sequences=[],
+    )
+
+
+def get_language_modeling_adapter_spec() -> AdapterSpec:
+    """
+    Used for language modeling.
+    """
+    return AdapterSpec(
+        method=ADAPT_LANGUAGE_MODELING,
+        instructions="",
+        input_prefix="",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        max_train_instances=0,
+        num_outputs=1,
+        max_tokens=0,
+        temperature=0.0,
+    )
+
+
+def get_summarization_adapter_spec(num_sents: Optional[int], max_train_instances: int = 5, **kwargs) -> AdapterSpec:
+    """
+    Used for summarization.
+    """
+
+    if num_sents == 1:
+        out_pref = "Summarize the above article in 1 sentence.\n"
+    elif num_sents is None:
+        out_pref = "Summarize the above article.\n"
+    else:
+        out_pref = f"Summarize the above article in {num_sents} sentences.\n"
+
+    return AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions="",
+        input_prefix="###\nArticle: ",
+        input_suffix="\n\n",
+        output_prefix=out_pref,
+        output_suffix="\n",
+        max_train_instances=max_train_instances,
+        num_outputs=1,
+        stop_sequences=["###"],  # Separator between few-shot instances.
+        **kwargs,
+    )
+
+
+def get_machine_translation_adapter_spec(
+    source_language, target_language, max_train_instances, **kwargs
+) -> AdapterSpec:
+    """
+    Used for machine translation.
+    """
+    return AdapterSpec(
+        method=ADAPT_GENERATION,
+        instructions=f"Translate the following sentences from {source_language} to {target_language}.",
+        input_prefix=f"{source_language}: ",
+        input_suffix="\n",
+        output_prefix=f"{target_language}: ",
+        output_suffix="\n",
+        max_train_instances=max_train_instances,
+        num_outputs=1,
+        stop_sequences=["\n\n"],
+        temperature=0.0,
+        **kwargs,
+    )

helm/benchmark/adaptation/request_state.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Optional, Dict, List
+from typing import Optional, Dict, List, Any
 
 from helm.benchmark.scenarios.scenario import Instance
 from helm.common.general import indent_lines, format_text_lines, serialize
@@ -45,6 +45,11 @@ class RequestState:
     num_conditioning_tokens: int = 0
     """The number of initial tokens that will be ignored when computing language modeling metrics"""
 
+    annotations: Optional[Dict[str, Any]] = None
+    """Output of some post-processing step that is needed for the metric to understand the request
+    Should match the annotator's name to an Annotation (usually a list of dictionaries for each completion)
+    Example: parsing, rendering an image based on the text completion, etc."""
+
     def __post_init__(self):
         if self.request_mode:
             assert self.request_mode in ["original", "calibration"], f"Invalid request_mode: {self.request_mode}"

helm/benchmark/adaptation/scenario_state.py

@@ -3,8 +3,9 @@ from dataclasses import dataclass
 from typing import List, Dict, Tuple, Optional
 
 from helm.benchmark.scenarios.scenario import Instance
-from .adapter_spec import AdapterSpec
-from .request_state import RequestState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.annotator import AnnotatorSpec
 
 
 @dataclass
@@ -21,6 +22,9 @@ class ScenarioState:
     # List of `RequestState`s that were produced by adaptation (and execution)
     request_states: List[RequestState]
 
+    # Annotations to use for this run spec
+    annotator_specs: Optional[List[AnnotatorSpec]] = None
+
     def __post_init__(self):
         # Create derived indices based on `request_states` so it's easier for
         # the `Metric` later to access them.  Two things are produced:

helm/benchmark/annotation/annotator.py

@@ -0,0 +1,43 @@
+from typing import Dict, List, Any
+from abc import abstractmethod, ABC
+from dataclasses import dataclass
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.common.object_spec import ObjectSpec
+
+
+class Annotator(ABC):
+    """Annotator is an abstract class for annotating a request state. Annotators are used to add additional
+    information to a request state that is needed for a metric to understand the request. This could be
+    parsing, rendering an image based on the text completion, etc."""
+
+    name: str
+    """Name of the annotator. Should be filled in by the subclass."""
+
+    @abstractmethod
+    def annotate(self, request_state: RequestState) -> Any:
+        """Fills the annotations field of the request state with additional information
+        that are implementation specific."""
+        pass
+
+
+@dataclass(frozen=True)
+class AnnotatorSpec(ObjectSpec):
+    """Specifies how to create an `Annotator`.
+    The user should only specify the class name.
+    The arguments will be filled in by the `AnnotatorFactory`.
+    """
+
+    pass
+
+
+class DummyAnnotator(Annotator):
+    """A dummy annotator that does nothing."""
+
+    name = "dummy"
+
+    def annotate(self, request_state: RequestState) -> List[Dict[str, Any]]:
+        if request_state.result is None:
+            raise ValueError("Annotation requires a result")
+        annotation_values: List[str] = [completion.text.upper() for completion in request_state.result.completions]
+        return [{"all_caps": value} for value in annotation_values]

helm/benchmark/annotation/annotator_factory.py

@@ -0,0 +1,61 @@
+import os
+from typing import Any, Dict, Mapping, Optional
+
+from helm.common.credentials_utils import provide_api_key
+from helm.common.cache_backend_config import CacheBackendConfig, CacheConfig
+from helm.common.hierarchical_logger import hlog
+from helm.common.object_spec import create_object, inject_object_spec_args
+from helm.benchmark.annotation.annotator import Annotator, AnnotatorSpec
+
+
+class AnnotatorFactory:
+    """Factory for creating annotators."""
+
+    def __init__(
+        self, credentials: Mapping[str, Any], file_storage_path: str, cache_backend_config: CacheBackendConfig
+    ):
+        self.credentials = credentials
+        self.file_storage_path = file_storage_path
+        self.cache_backend_config = cache_backend_config
+        hlog(f"AnnotatorFactory: file_storage_path = {file_storage_path}")
+        hlog(f"AnnotatorFactory: cache_backend_config = {cache_backend_config}")
+
+        # Cache for annotators
+        # This is used to prevent duplicate creation of annotators
+        # It is especially important as annotation is a multi-threaded
+        # process and creating a new annotator for each request can cause
+        # race conditions.
+        self.annotators: Dict[str, Annotator] = {}
+
+    def get_annotator(self, annotator_spec: AnnotatorSpec) -> Annotator:
+        """Return a annotator based on the name."""
+        # First try to find the annotator in the cache
+        assert annotator_spec.args is None or annotator_spec.args == {}
+        annotator_name: str = annotator_spec.class_name.split(".")[-1].lower().replace("annotator", "")
+        annotator: Optional[Annotator] = self.annotators.get(annotator_name)
+        if annotator is not None:
+            return annotator
+
+        # Otherwise, create the client
+        cache_config: CacheConfig = self.cache_backend_config.get_cache_config(annotator_name)
+        annotator_spec = inject_object_spec_args(
+            annotator_spec,
+            constant_bindings={
+                "cache_config": cache_config,
+            },
+            provider_bindings={
+                "api_key": lambda: provide_api_key(self.credentials, annotator_name),
+                "file_storage_path": lambda: self._get_file_storage_path(annotator_name),
+            },
+        )
+        annotator = create_object(annotator_spec)
+
+        # Cache the client
+        self.annotators[annotator_name] = annotator
+
+        return annotator
+
+    def _get_file_storage_path(self, annotator_name: str) -> str:
+        # Returns the path to use for a local file cache for the given annotator
+        local_file_cache_path: str = os.path.join(self.file_storage_path, "output", annotator_name)
+        return local_file_cache_path

helm/benchmark/annotation/image2structure/image_compiler_annotator.py

@@ -0,0 +1,88 @@
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Tuple, Callable
+
+from dacite import from_dict
+
+from helm.benchmark.annotation.annotator import Annotator
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.common.cache import Cache, CacheConfig
+from helm.common.file_caches.local_file_cache import LocalPILFileCache
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.common.media_object import MediaObject
+from helm.proxy.retry import get_retry_decorator
+
+try:
+    from PIL import Image
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, suggestions=["images"])
+
+
+def retry_if_compilation_failed(result: Dict[str, Any]) -> bool:
+    """Retries when the compilation fails."""
+    return "unknown_error" in result
+
+
+retry: Callable = get_retry_decorator(
+    "Compilation", max_attempts=5, wait_exponential_multiplier_seconds=2, retry_on_result=retry_if_compilation_failed
+)
+
+
+class CompilationError(Exception):
+    pass
+
+
+class ImageCompilerAnnotator(Annotator, ABC):
+    """Annotator that compiles the text completions into an image."""
+
+    def __init__(self, cache_config: CacheConfig, file_storage_path: str):
+        self._cache = Cache(cache_config)
+        self._file_cache = LocalPILFileCache(file_storage_path)
+
+    @abstractmethod
+    def compile_completion_into_image(
+        self, request_state: RequestState, completion_text: str
+    ) -> Tuple[Image.Image, Dict[str, Any]]:
+        raise NotImplementedError
+
+    def postprocess_infos(self, infos: Dict[str, Any]) -> Dict[str, Any]:
+        """Postprocess the infos."""
+        return infos
+
+    def annotate(self, request_state: RequestState) -> List[Dict[str, Any]]:
+        """Fills the annotations field of the request state with the compiled image."""
+        assert request_state.result is not None, "Annotator can only be used after the request has been processed."
+        annotations: List[Dict[str, Any]] = []
+        for completion in request_state.result.completions:
+            completion_text: str = completion.text.strip()
+            raw_response: Dict[str, Any]
+
+            @retry
+            def compile() -> Dict[str, Any]:
+                def do_it() -> Dict[str, Any]:
+                    try:
+                        assert self._file_cache is not None
+                        image, infos = self.compile_completion_into_image(request_state, completion_text)
+                        infos = self.postprocess_infos(infos)
+                        image_path: str = self._file_cache.store_image(lambda: image)
+                        return {
+                            "media_object": MediaObject(location=image_path, content_type="image/png").to_dict(),
+                            **infos,
+                        }
+                    except CompilationError as e:
+                        return {"error": str(e)}
+
+                try:
+                    cache_key: Dict[str, str] = {"completion": completion_text}
+                    raw_response, _ = self._cache.get(cache_key, do_it)
+                    return raw_response
+                except Exception as e:
+                    return {"unknown_error": str(e)}
+
+            raw_response = compile()
+            response = {**raw_response}
+            if "media_object" in response:
+                response["media_object"] = from_dict(MediaObject, response["media_object"])
+
+            # Merge annotations
+            annotations.append(response)
+        return annotations

helm/benchmark/annotation/image2structure/latex_compiler_annotator.py

@@ -0,0 +1,59 @@
+from typing import List, Tuple, Dict, Any
+
+from helm.benchmark.annotation.image2structure.image_compiler_annotator import ImageCompilerAnnotator, CompilationError
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.benchmark.scenarios.vision_language.image2structure.utils_latex import (
+    latex_to_image,
+    strip_unnecessary_latex_parts,
+)
+
+try:
+    from PIL import Image
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, suggestions=["images"])
+
+
+class LatexCompilerAnnotator(ImageCompilerAnnotator):
+    """Annotator that compiles the text completions into a LaTeX document."""
+
+    name: str = "latex_compiler"
+
+    # Delimiters for the code block
+    DELIMITERS: List[Tuple[str, str]] = [
+        ("```latex", "```"),
+        ("```", "```"),
+    ]
+
+    def postprocess_infos(self, infos: Dict[str, Any]) -> Dict[str, Any]:
+        """Postprocess the infos."""
+        annotations = super().postprocess_infos(infos)
+        assert "latex_code" in annotations, "The latex_code field should be present in the infos"
+        annotations["text"] = strip_unnecessary_latex_parts(annotations["latex_code"])
+        return annotations
+
+    def compile_completion_into_image(
+        self, request_state: RequestState, completion_text: str
+    ) -> Tuple[Image.Image, Dict[str, Any]]:
+        """Given a completion, parse the LaTeX and compile it into an image."""
+        # Get the assets path
+        assets_path: str = ""
+
+        # Check for code block delimiters
+        # After this completion should be a valid latex code block
+        for start, end in self.DELIMITERS:
+            if start in completion_text and end in completion_text[completion_text.index(start) + len(start) :]:
+                start_index = completion_text.index(start) + len(start)
+                end_index = completion_text.index(end, start_index)
+                completion_text = completion_text[start_index:end_index]
+                break
+
+        # Convert the latex code to an image
+        try:
+            image, infos = latex_to_image(completion_text, assets_path, crop=True)
+        except RuntimeError as e:
+            # We do not want to catch OptionalDependencyNotInstalled (error with latex installation)
+            # because it is a fatal error and should be handled by the user
+            raise CompilationError(str(e)) from e
+
+        return image, infos