crfm-helm 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/vlm_run_specs.py

@@ -1,97 +0,0 @@
-from typing import List, Optional
-
-from .adaptation.adapter_spec import AdapterSpec
-from .adaptation.adapters.adapter_factory import ADAPT_GENERATION_MULTIMODAL
-from .metrics.metric import MetricSpec
-from .run_specs import run_spec_function, get_exact_match_metric_specs
-from .runner import RunSpec
-from .scenarios.scenario import ScenarioSpec
-
-
-############################################################
-# Prototypical adapter specs for VLM evaluation
-
-
-def get_vlm_generation_adapter_spec(
-    instructions: str = "",
-    input_prefix: str = "",
-    input_suffix: str = "",
-    output_prefix: str = "",
-    output_suffix: str = "",
-    max_tokens: int = 100,
-    stop_sequences: Optional[List[str]] = None,
-) -> AdapterSpec:
-    return AdapterSpec(
-        method=ADAPT_GENERATION_MULTIMODAL,
-        global_prefix="",
-        instructions=instructions,
-        input_prefix=input_prefix,
-        input_suffix=input_suffix,
-        output_prefix=output_prefix,
-        output_suffix=output_suffix,
-        instance_prefix="\n",
-        # We focus on zero-shot evaluation for now as most open VLMs only support a single image input
-        max_train_instances=0,
-        num_outputs=1,
-        max_tokens=max_tokens,
-        stop_sequences=stop_sequences if stop_sequences is not None else [],
-        random=None,
-    )
-
-
-############################################################
-# VHELM run specs
-
-
-@run_spec_function("viz_wiz")
-def get_viz_wiz_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.vision_language.viz_wiz_scenario.VizWizScenario", args={}
-    )
-
-    # TODO: finalize the adapter spec parameters once we add more models
-    adapter_spec: AdapterSpec = get_vlm_generation_adapter_spec(
-        input_prefix="User: ",
-        input_suffix="<end_of_utterance>",
-        output_prefix="\nAssistant: ",
-        output_suffix="<end_of_utterance>",
-        stop_sequences=["<end_of_utterance>"],
-    )
-
-    metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
-
-    run_spec_name: str = "viz_wiz"
-    return RunSpec(
-        name=run_spec_name,
-        scenario_spec=scenario_spec,
-        adapter_spec=adapter_spec,
-        metric_specs=metric_specs,
-        groups=[run_spec_name],
-    )
-
-
-@run_spec_function("vqa")
-def get_vqa_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.vision_language.vqa_scenario.VQAScenario", args={}
-    )
-
-    # TODO: finalize the adapter spec parameters once we add more models
-    adapter_spec: AdapterSpec = get_vlm_generation_adapter_spec(
-        input_prefix="User: ",
-        input_suffix="<end_of_utterance>",
-        output_prefix="\nAssistant: ",
-        output_suffix="<end_of_utterance>",
-        stop_sequences=["<end_of_utterance>"],
-    )
-
-    metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
-
-    run_spec_name: str = "vqa"
-    return RunSpec(
-        name=run_spec_name,
-        scenario_spec=scenario_spec,
-        adapter_spec=adapter_spec,
-        metric_specs=metric_specs,
-        groups=[run_spec_name],
-    )

helm/benchmark/window_services/flan_t5_window_service.py

@@ -1,29 +0,0 @@
-from .encoder_decoder_window_service import EncoderDecoderWindowService
-from .tokenizer_service import TokenizerService
-
-
-class FlanT5WindowService(EncoderDecoderWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length."""
-        # We subtract 1 to account for <extra_id_0> that gets appended to prompts.
-        return 512 - 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "</s>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "google/flan-t5-xxl"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token is the same as the end of text token."""
-        # echo=True is not supported
-        return ""

helm/benchmark/window_services/gpt2_window_service.py

@@ -1,32 +0,0 @@
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class GPT2WindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length of this tokenizer."""
-        return 1024
-
-    @property
-    def max_request_length(self) -> int:
-        """Return the max request length of GPT-2."""
-        return self.max_sequence_length + 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "<|endoftext|>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "huggingface/gpt2"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token for models that uses the GPT-2 tokenizer is the end of text token."""
-        return self.end_of_text_token

helm/benchmark/window_services/huggingface_window_service.py

@@ -1,60 +0,0 @@
-from typing import Optional
-from helm.proxy.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class HuggingFaceWindowService(LocalWindowService):
-    def __init__(
-        self,
-        service: TokenizerService,
-        tokenizer_name: str,
-        pretrained_model_name_or_path: Optional[str] = None,
-        max_sequence_length: Optional[int] = None,
-        max_request_length: Optional[int] = None,
-        end_of_text_token: Optional[str] = None,
-        prefix_token: Optional[str] = None,
-        **kwargs
-    ):
-        super().__init__(service)
-        self._tokenizer_name = tokenizer_name
-        # Override max_sequence_length, max_request_length, end_of_text_token
-        # and prefix_token if provided as an argument.
-        # Otherwise, auto-infer them from the Hugging Face tokenizer.
-        #
-        # Note that many Hugging Face tokenizers have incorrect sequence lengths,
-        # so it is recommended to set this manually.
-        with HuggingFaceTokenizer.get_tokenizer(
-            helm_tokenizer_name=tokenizer_name,
-            pretrained_model_name_or_path=pretrained_model_name_or_path or tokenizer_name,
-            **kwargs,
-        ) as tokenizer:
-            self._max_sequence_length = max_sequence_length or tokenizer.model_max_length
-            self._max_request_length = max_request_length or self._max_sequence_length
-            self._end_of_text_token = end_of_text_token or tokenizer.eos_token or ""
-            self._prefix_token = prefix_token or tokenizer.bos_token or ""
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return self._tokenizer_name
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length of this tokenizer."""
-        return self._max_sequence_length
-
-    @property
-    def max_request_length(self) -> int:
-        """Return the max request length of this tokenizer."""
-        return self._max_request_length
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return self._end_of_text_token
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token."""
-        return self._prefix_token

helm/benchmark/window_services/t0pp_window_service.py

@@ -1,35 +0,0 @@
-from .encoder_decoder_window_service import EncoderDecoderWindowService
-from .tokenizer_service import TokenizerService
-
-
-class T0ppWindowService(EncoderDecoderWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length."""
-        # From https://arxiv.org/pdf/2110.08207.pdf, "we truncate input and target sequences to 1024 and 256 tokens,
-        # respectively. Following Raffel et al. (2020), we use packing to combine multiple training examples into
-        # a single sequence to reach the maximum sequence length."
-        return 1024
-
-    @property
-    def max_output_length(self) -> int:
-        return 256
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "</s>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "bigscience/T0pp"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token is the same as the end of text token."""
-        # echo=True is not supported
-        return ""

helm/benchmark/window_services/t511b_window_service.py

@@ -1,30 +0,0 @@
-from .encoder_decoder_window_service import EncoderDecoderWindowService
-from .tokenizer_service import TokenizerService
-
-
-class T511bWindowService(EncoderDecoderWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length."""
-        # From https://arxiv.org/pdf/1910.10683.pdf, "we use a maximum sequence length of 512".
-        # We subtract 1 to account for <extra_id_0> that gets appended to prompts.
-        return 512 - 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "</s>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "google/t5-11b"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token is the same as the end of text token."""
-        # echo=True is not supported
-        return ""

helm/benchmark/window_services/test_mt_nlg_window_service.py

@@ -1,48 +0,0 @@
-import shutil
-import tempfile
-
-from .test_utils import get_tokenizer_service, TEST_PROMPT, GPT2_TEST_TOKENS, GPT2_TEST_TOKEN_IDS
-from .tokenizer_service import TokenizerService
-from .window_service_factory import WindowServiceFactory
-
-
-class TestMTNLGWindowService:
-    def setup_method(self):
-        self.path: str = tempfile.mkdtemp()
-        service: TokenizerService = get_tokenizer_service(self.path)
-        self.window_service = WindowServiceFactory.get_window_service("microsoft/TNLGv2_7B", service)
-
-    def teardown_method(self, method):
-        shutil.rmtree(self.path)
-
-    def test_max_request_length(self):
-        assert self.window_service.max_request_length == 2048
-
-    def test_encode(self):
-        assert self.window_service.encode(TEST_PROMPT).token_values == GPT2_TEST_TOKEN_IDS
-
-    def test_decode(self):
-        assert self.window_service.decode(self.window_service.encode(TEST_PROMPT).tokens) == TEST_PROMPT
-
-    def test_tokenize(self):
-        assert self.window_service.tokenize(TEST_PROMPT) == GPT2_TEST_TOKENS
-
-    def test_fits_within_context_window(self):
-        # Should fit in the context window since we subtracted the number of tokens of the test prompt
-        # from the max request length of 2048
-        assert self.window_service.fits_within_context_window(TEST_PROMPT, 2048 - 51)
-        # Should not fit within the max request length because we're expecting one more extra token in the completion
-        assert not self.window_service.fits_within_context_window(TEST_PROMPT, 2048 - 51 + 1)
-
-    def test_truncate_from_right(self):
-        # Create a prompt that exceed max context length: 51 * 41 = 2091 tokens
-        long_prompt: str = TEST_PROMPT * 41
-        assert not self.window_service.fits_within_context_window(long_prompt)
-
-        # Truncate and ensure it fits within the context window
-        truncated_long_prompt: str = self.window_service.truncate_from_right(long_prompt)
-        assert self.window_service.get_num_tokens(truncated_long_prompt) == 2048
-        assert self.window_service.fits_within_context_window(truncated_long_prompt)
-
-    def test_tokenize_and_count(self):
-        assert self.window_service.get_num_tokens(TEST_PROMPT) == 51

helm/benchmark/window_services/ul2_window_service.py

@@ -1,30 +0,0 @@
-from .encoder_decoder_window_service import EncoderDecoderWindowService
-from .tokenizer_service import TokenizerService
-
-
-class UL2WindowService(EncoderDecoderWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length."""
-        # From https://arxiv.org/pdf/2205.05131.pdf, "the sequence length is set to 512/512 for inputs and targets".
-        # We subtract 1 to account for <extra_id_0> that gets appended to prompts.
-        return 512 - 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "</s>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "google/ul2"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token is the same as the end of text token."""
-        # echo=True is not supported
-        return ""

helm/benchmark/window_services/wider_ai21_window_service.py

@@ -1,24 +0,0 @@
-from .ai21_window_service import AI21WindowService
-
-
-class WiderAI21WindowService(AI21WindowService):
-    @property
-    def max_sequence_length(self) -> int:
-        """
-        Return the max sequence length of the larger AI21 Jurassic-2 models.
-
-        The AI21 server automatically prepends a token to every prompt,
-        so the actual max sequence length is 8192 - 1 = 8191.
-        """
-        return 8191
-
-
-class AI21Jurassic2JumboWindowService(AI21WindowService):
-    @property
-    def max_sequence_length(self) -> int:
-        """
-        Return the max sequence length of the AI21 Jurassic-2 Jumbo.
-
-        AI21 has recommended using a sequence length of 6000 tokens to avoid OOMs.
-        """
-        return 6000

helm/common/cache_utils.py

@@ -1,14 +0,0 @@
-"""Functions used for caching."""
-
-import os
-
-from helm.common.cache import CacheConfig, MongoCacheConfig, SqliteCacheConfig
-
-
-def build_cache_config(cache_path: str, mongo_uri: str, organization: str) -> CacheConfig:
-    if mongo_uri:
-        return MongoCacheConfig(mongo_uri, collection_name=organization)
-
-    client_cache_path: str = os.path.join(cache_path, f"{organization}.sqlite")
-    # TODO: Allow setting CacheConfig.follower_cache_path from a command line flag.
-    return SqliteCacheConfig(client_cache_path)

helm/proxy/clients/aleph_alpha_client.py

@@ -1,95 +0,0 @@
-import json
-import requests
-from typing import Any, Dict, List
-
-from helm.common.cache import CacheConfig
-from helm.common.request import wrap_request_time, Request, RequestResult, Sequence, Token
-from .client import CachingClient, truncate_sequence
-
-
-class AlephAlphaClient(CachingClient):
-    COMPLETION_ENDPOINT: str = "complete"
-
-    def __init__(self, api_key: str, cache_config: CacheConfig):
-        super().__init__(cache_config=cache_config)
-        self.api_key: str = api_key
-
-    def _send_request(self, endpoint: str, raw_request: Dict[str, Any]) -> Dict[str, Any]:
-        response = requests.request(
-            method="POST",
-            url=f"https://api.aleph-alpha.com/{endpoint}",
-            headers={
-                "Content-Type": "application/json",
-                "Accept": "application/json",
-                "Authorization": f"Bearer {self.api_key}",
-            },
-            data=json.dumps(raw_request),
-            # Setting the nice flag prevents intensive benchmarking runs from saturating Aleph Alpha's API queues
-            params=json.dumps({"nice": True}),
-        )
-        result = json.loads(response.text)
-        assert "error" not in result, f"Request failed with error: {result['error']}"
-        return result
-
-    def make_request(self, request: Request) -> RequestResult:
-        """Make a request following https://docs.aleph-alpha.com/api/complete."""
-        raw_request = {
-            "model": request.model_engine,
-            "prompt": request.prompt,
-            "maximum_tokens": request.max_tokens,
-            "temperature": request.temperature,
-            "top_k": request.top_k_per_token,
-            "top_p": request.top_p,
-            "presence_penalty": request.presence_penalty,
-            "frequency_penalty": request.frequency_penalty,
-            "n": request.num_completions,
-            "stop_sequences": request.stop_sequences,
-            "log_probs": request.top_k_per_token,
-            "echo": request.echo_prompt,
-            "tokens": True,  # Setting to True returns individual tokens of the completion
-        }
-
-        try:
-
-            def do_it():
-                result = self._send_request(AlephAlphaClient.COMPLETION_ENDPOINT, raw_request)
-                assert "completions" in result, f"Invalid response: {result}"
-                return result
-
-            response, cached = self.cache.get(raw_request, wrap_request_time(do_it))
-        except (requests.exceptions.RequestException, AssertionError) as e:
-            error: str = f"AlephAlphaClient error: {e}"
-            return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
-
-        completions: List[Sequence] = []
-        for completion in response["completions"]:
-            sequence_logprob: float = 0
-            tokens: List[Token] = []
-
-            # `completion_tokens` is the list of selected tokens.
-            for i, token in enumerate(completion.get("completion_tokens", [])):
-                # Get the top K logprobs for the ith token
-                top_logprobs: Dict[str, float] = completion["log_probs"][i]
-                # Use the selected token value to get the logprob
-                logprob: float = top_logprobs[token]
-                sequence_logprob += logprob
-                tokens.append(
-                    Token(
-                        text=token,
-                        logprob=logprob,
-                        top_logprobs=top_logprobs,
-                    )
-                )
-
-            sequence: Sequence = Sequence(text=completion["completion"], logprob=sequence_logprob, tokens=tokens)
-            sequence = truncate_sequence(sequence, request)
-            completions.append(sequence)
-
-        return RequestResult(
-            success=True,
-            cached=cached,
-            request_time=response["request_time"],
-            request_datetime=response["request_datetime"],
-            completions=completions,
-            embedding=[],
-        )

helm/proxy/clients/goose_ai_client.py

@@ -1,99 +0,0 @@
-from typing import List, Optional
-
-import openai as gooseai
-
-from helm.common.cache import CacheConfig
-from helm.common.request import (
-    wrap_request_time,
-    EMBEDDING_UNAVAILABLE_REQUEST_RESULT,
-    Request,
-    RequestResult,
-    Sequence,
-    Token,
-)
-from .client import CachingClient, truncate_sequence
-from .openai_client import ORIGINAL_COMPLETION_ATTRIBUTES
-
-
-class GooseAIClient(CachingClient):
-    """
-    GooseAI API Client
-    - How to use the API: https://goose.ai/docs/api
-    - Supported models: https://goose.ai/docs/models
-    """
-
-    def __init__(self, api_key: str, cache_config: CacheConfig, org_id: Optional[str] = None):
-        super().__init__(cache_config=cache_config)
-        self.org_id: Optional[str] = org_id
-        self.api_key: str = api_key
-        self.api_base: str = "https://api.goose.ai/v1"
-
-    def make_request(self, request: Request) -> RequestResult:
-        """
-        Request parameters for GooseAI API documented here: https://goose.ai/docs/api/completions
-        The only OpenAI API parameter not supported is `best_of`.
-        """
-        # Embedding not supported for this model
-        if request.embedding:
-            return EMBEDDING_UNAVAILABLE_REQUEST_RESULT
-
-        raw_request = {
-            "engine": request.model_engine,
-            "prompt": request.prompt,
-            "temperature": request.temperature,
-            "n": request.num_completions,
-            "max_tokens": request.max_tokens,
-            "logprobs": request.top_k_per_token,
-            "stop": request.stop_sequences or None,  # API doesn't like empty list
-            "top_p": request.top_p,
-            "presence_penalty": request.presence_penalty,
-            "frequency_penalty": request.frequency_penalty,
-            "echo": request.echo_prompt,
-        }
-
-        try:
-
-            def do_it():
-                # Following https://beta.openai.com/docs/api-reference/authentication
-                # `organization` can be set to None.
-                gooseai.organization = self.org_id
-                gooseai.api_key = self.api_key
-                gooseai.api_base = self.api_base
-                gooseai.api_resources.completion.Completion.__bases__ = ORIGINAL_COMPLETION_ATTRIBUTES
-                return gooseai.Completion.create(**raw_request)
-
-            cache_key = CachingClient.make_cache_key(raw_request, request)
-            response, cached = self.cache.get(cache_key, wrap_request_time(do_it))
-        except gooseai.error.OpenAIError as e:
-            error: str = f"OpenAI (GooseAI API) error: {e}"
-            return RequestResult(success=False, cached=False, error=error, completions=[], embedding=[])
-
-        completions: List[Sequence] = []
-        for raw_completion in response["choices"]:
-            sequence_logprob = 0
-            tokens: List[Token] = []
-
-            raw_data = raw_completion["logprobs"]
-            for text, logprob, top_logprobs in zip(
-                raw_data["tokens"], raw_data["token_logprobs"], raw_data["top_logprobs"]
-            ):
-                tokens.append(Token(text=text, logprob=logprob or 0, top_logprobs=dict(top_logprobs or {})))
-                sequence_logprob += logprob or 0
-
-            completion = Sequence(
-                text=raw_completion["text"],
-                logprob=sequence_logprob,
-                tokens=tokens,
-                finish_reason={"reason": raw_completion["finish_reason"]},
-            )
-            completion = truncate_sequence(completion, request)
-            completions.append(completion)
-
-        return RequestResult(
-            success=True,
-            cached=cached,
-            request_time=response["request_time"],
-            request_datetime=response.get("request_datetime"),
-            completions=completions,
-            embedding=[],
-        )