crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/window_services/gpt2_window_service.py

@@ -1,32 +0,0 @@
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class GPT2WindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length of this tokenizer."""
-        return 1024
-
-    @property
-    def max_request_length(self) -> int:
-        """Return the max request length of GPT-2."""
-        return self.max_sequence_length + 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "<|endoftext|>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "huggingface/gpt2"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token for models that uses the GPT-2 tokenizer is the end of text token."""
-        return self.end_of_text_token

helm/benchmark/window_services/gptj_window_service.py

@@ -1,38 +0,0 @@
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class GPTJWindowService(LocalWindowService):
-    """
-    The same tokenizer as GPT-2, but with an additional 143 tokens
-    (source: https://huggingface.co/docs/transformers/model_doc/gptj).
-    """
-
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length."""
-        return 2048
-
-    @property
-    def max_request_length(self) -> int:
-        """Return the max request length."""
-        return self.max_sequence_length + 1
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        # Not a typo: Named "gpt-j-6B" instead of "gpt-j-6b" in Hugging Face
-        return "EleutherAI/gpt-j-6B"
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "<|endoftext|>"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token for models is the same as the end of text token."""
-        return self.end_of_text_token

helm/benchmark/window_services/gptneox_window_service.py

@@ -1,41 +0,0 @@
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class GPTNeoXWindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length."""
-        return 2048
-
-    @property
-    def max_request_length(self) -> int:
-        """Return the max request length."""
-        return self.max_sequence_length + 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "<|endoftext|>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "EleutherAI/gpt-neox-20b"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token is the same as the end of text token."""
-        return self.end_of_text_token
-
-
-class StableLMAlphaWindowService(GPTNeoXWindowService):
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length."""
-        # The context length for these models is 4096 tokens.
-        # See: https://github.com/Stability-AI/StableLM#stablelm-alpha
-        return 4096

helm/benchmark/window_services/http_model_window_service.py

@@ -1,28 +0,0 @@
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-# TODO: Remove Once we have configurable model names since this hardcodes the tokenizer name
-class HTTPModelWindowServce(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        return 2048
-
-    @property
-    def max_request_length(self) -> int:
-        return self.max_sequence_length
-
-    @property
-    def end_of_text_token(self) -> str:
-        return "<|endoftext|>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        return "neurips/local"
-
-    @property
-    def prefix_token(self) -> str:
-        return self.end_of_text_token

helm/benchmark/window_services/huggingface_window_service.py

@@ -1,59 +0,0 @@
-from typing import Optional
-from helm.proxy.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class HuggingFaceWindowService(LocalWindowService):
-    def __init__(
-        self,
-        service: TokenizerService,
-        tokenizer_name: str,
-        pretrained_model_name_or_path: Optional[str] = None,
-        revision: Optional[str] = None,
-        max_sequence_length: Optional[int] = None,
-        max_reqeust_length: Optional[int] = None,
-    ):
-        super().__init__(service)
-        self._tokenizer_name = tokenizer_name
-        tokenizer = HuggingFaceTokenizer.get_tokenizer(
-            helm_tokenizer_name=tokenizer_name,
-            pretrained_model_name_or_path=pretrained_model_name_or_path or tokenizer_name,
-            revision=revision,
-        )
-        self._prefix_token = tokenizer.bos_token
-        self._end_of_text_token = tokenizer.eos_token
-        # Override max_sequence_length if provided as an argument.
-        # Otherwise, auto-infer max_sequence_length from the Hugging Face tokenizer.
-        # Note that many Hugging Face tokenizers have incorrect sequence lengths,
-        # so it is recommended to set this manually.
-        if max_sequence_length:
-            self._max_sequence_length = max_sequence_length
-        else:
-            self._max_sequence_length = tokenizer.model_max_length
-        self._max_request_length = max_reqeust_length
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length of this tokenizer."""
-        return self._max_sequence_length
-
-    @property
-    def max_request_length(self) -> int:
-        """Return the max request length of this tokenizer."""
-        return self._max_request_length or self._max_sequence_length
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return self._end_of_text_token
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return self._tokenizer_name
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token."""
-        return self._prefix_token

helm/benchmark/window_services/lit_gpt_window_service.py

@@ -1,27 +0,0 @@
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class LitGPTWindowServce(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        return 2048
-
-    @property
-    def max_request_length(self) -> int:
-        return self.max_sequence_length
-
-    @property
-    def end_of_text_token(self) -> str:
-        return "<|endoftext|>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        return "lightningai/lit-gpt"
-
-    @property
-    def prefix_token(self) -> str:
-        return self.end_of_text_token

helm/benchmark/window_services/llama_window_service.py

@@ -1,28 +0,0 @@
-from helm.benchmark.window_services.huggingface_window_service import HuggingFaceWindowService
-from helm.benchmark.window_services.tokenizer_service import TokenizerService
-
-
-class LlamaWindowService(HuggingFaceWindowService):
-    def __init__(self, service: TokenizerService):
-        # Tokenizer name hf-internal-testing/llama-tokenizer is taken from:
-        # https://huggingface.co/docs/transformers/main/en/model_doc/llama#transformers.LlamaTokenizerFast.example
-        super().__init__(service, tokenizer_name="hf-internal-testing/llama-tokenizer")
-
-
-class Llama2WindowService(HuggingFaceWindowService):
-    # To use the Llama-2 tokenizer:
-    #
-    # 1. Accept the license agreement: https://ai.meta.com/resources/models-and-libraries/llama-downloads/
-    # 2. Request to access the Hugging Face repository: https://huggingface.co/meta-llama/Llama-2-7b
-    # 3. Run `huggingface-cli login`
-    #
-    # If you encounter the following error, complete the above steps and try again:
-    #
-    #     meta-llama/Llama-2-70b-hf is not a local folder and is not a valid model identifier listed on
-    #     'https://huggingface.co/models'
-    def __init__(self, service: TokenizerService):
-        super().__init__(service, "meta-llama/Llama-2-7b-hf")
-
-    @property
-    def max_sequence_length(self) -> int:
-        return 4096

helm/benchmark/window_services/luminous_window_service.py

@@ -1,67 +0,0 @@
-from abc import abstractmethod
-
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class LuminousWindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    @abstractmethod
-    def tokenizer_name(self) -> str:
-        """Each Luminous model has its own tokenizer."""
-        pass
-
-    @property
-    def max_sequence_length(self) -> int:
-        """
-        From https://docs.aleph-alpha.com/api/complete, "the summed number of tokens of prompt
-        and maximum_tokens..may not exceed 2048 tokens." Confirmed it's 2048 for the Luminous
-        models currently available.
-        """
-        return 2048
-
-    @property
-    def max_request_length(self) -> int:
-        return self.max_sequence_length
-
-    @property
-    def end_of_text_token(self) -> str:
-        """
-        The end of text token.
-        TODO: Setting to empty string for now as echo is not supported.
-        """
-        return ""
-
-    @property
-    def prefix_token(self) -> str:
-        """
-        The prefix token.
-        """
-        return self.end_of_text_token
-
-
-class LuminousBaseWindowService(LuminousWindowService):
-    @property
-    def tokenizer_name(self) -> str:
-        return "AlephAlpha/luminous-base"
-
-
-class LuminousExtendedWindowService(LuminousWindowService):
-    @property
-    def tokenizer_name(self) -> str:
-        return "AlephAlpha/luminous-extended"
-
-
-class LuminousSupremeWindowService(LuminousWindowService):
-    @property
-    def tokenizer_name(self) -> str:
-        return "AlephAlpha/luminous-supreme"
-
-
-class LuminousWorldWindowService(LuminousWindowService):
-    @property
-    def tokenizer_name(self) -> str:
-        return "AlephAlpha/luminous-world"

helm/benchmark/window_services/megatron_window_service.py

@@ -1,10 +0,0 @@
-from .gpt2_window_service import GPT2WindowService
-
-
-# NOTE: The only difference between this and GPT2WindowService is that
-# the request length is constrained to the sequence length.
-class MegatronWindowService(GPT2WindowService):
-    @property
-    def max_request_length(self) -> int:
-        """Return the max request length of GPT-2."""
-        return self.max_sequence_length

helm/benchmark/window_services/mt_nlg_window_service.py

@@ -1,27 +0,0 @@
-from .gpt2_window_service import GPT2WindowService
-from .tokenizer_service import TokenizerService
-
-
-class MTNLGWindowService(GPT2WindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """
-        The max length of the model input. MT-NLG does not predict the logprob of the first
-        input token so `max_sequence_length` is one token shorter than `max_request_length`.
-        """
-        return self.max_request_length - 1
-
-    @property
-    def max_request_length(self) -> int:
-        """
-        The max request length for the MT-NLG models is 2048.
-        Source: https://github.com/microsoft/turing-academic-TNLG
-        """
-        return 2048
-
-    @property
-    def prefix_token(self) -> str:
-        return "<<"

helm/benchmark/window_services/openai_window_service.py

@@ -1,13 +0,0 @@
-from .gpt2_window_service import GPT2WindowService
-from .tokenizer_service import TokenizerService
-
-
-class OpenAIWindowService(GPT2WindowService):
-    def __init__(self, service: TokenizerService):
-        # OpenAI uses the same tokenizer for GPT-2 and GPT-3.
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length of the OpenAI models (max length of model input)."""
-        return 2048

helm/benchmark/window_services/opt_window_service.py

@@ -1,35 +0,0 @@
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class OPTWindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """
-        The max length of the model input. The max sequence length for the OPT models is 2048.
-        Source: https://arxiv.org/pdf/2205.01068.pdf
-        """
-        return 2048
-
-    @property
-    def max_request_length(self) -> int:
-        """Return the max request length."""
-        return self.max_sequence_length + 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "</s>"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token is the same as the end of text token."""
-        return self.end_of_text_token
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "facebook/opt-66b"

helm/benchmark/window_services/palmyra_window_service.py

@@ -1,45 +0,0 @@
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class PalmyraWindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def tokenizer_name(self) -> str:
-        """All Palmyra models use the same tokenizer."""
-        return "huggingface/gpt2"
-
-    @property
-    def max_sequence_length(self) -> int:
-        return 2048
-
-    @property
-    def max_request_length(self) -> int:
-        return self.max_sequence_length
-
-    @property
-    def max_sequence_and_generated_tokens_length(self) -> int:
-        return self.max_sequence_length
-
-    @property
-    def end_of_text_token(self) -> str:
-        """
-        The end of text token.
-        TODO: Setting to empty string for now as echo is not supported.
-        """
-        return ""
-
-    @property
-    def prefix_token(self) -> str:
-        """
-        The prefix token.
-        """
-        return self.end_of_text_token
-
-
-class LongerPalmyraWindowService(PalmyraWindowService):
-    @property
-    def max_sequence_length(self) -> int:
-        return 8192

helm/benchmark/window_services/remote_window_service.py

@@ -1,48 +0,0 @@
-from typing import Dict, Type
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class RemoteWindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService, model_name: str):
-        super().__init__(service)
-        self.model_name = model_name
-        info = self.service.get_info(model_name)
-        self._tokenizer_name = info.tokenizer_name
-        self._max_sequence_length = info.max_sequence_length
-        self._max_request_length = info.max_request_length
-        self._end_of_text_token = info.end_of_text_token
-        self._prefix_token = info.prefix_token
-
-    @property
-    def max_sequence_length(self) -> int:
-        return self._max_sequence_length
-
-    @property
-    def max_request_length(self) -> int:
-        return self._max_request_length
-
-    @property
-    def end_of_text_token(self) -> str:
-        return self._end_of_text_token
-
-    @property
-    def prefix_token(self) -> str:
-        return self._prefix_token
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return self._tokenizer_name
-
-
-# If the windowing logic is different from the base LocalWindowService,
-# please add the specific implementation for the model and add it to the following dict.
-remote_window_services: Dict[str, Type[RemoteWindowService]] = {}
-
-
-def get_remote_window_service(service: TokenizerService, model_name: str):
-    if model_name in remote_window_services:
-        return remote_window_services[model_name](service, model_name)
-    else:
-        return RemoteWindowService(service, model_name)

helm/benchmark/window_services/santacoder_window_service.py

@@ -1,27 +0,0 @@
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class SantaCoderWindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        return 2048
-
-    @property
-    def max_request_length(self) -> int:
-        return self.max_sequence_length
-
-    @property
-    def end_of_text_token(self) -> str:
-        return "<|endoftext|>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        return "bigcode/santacoder"
-
-    @property
-    def prefix_token(self) -> str:
-        return self.end_of_text_token

helm/benchmark/window_services/starcoder_window_service.py

@@ -1,27 +0,0 @@
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class StarCoderWindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        return 8192
-
-    @property
-    def max_request_length(self) -> int:
-        return self.max_sequence_length
-
-    @property
-    def end_of_text_token(self) -> str:
-        return "<|endoftext|>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        return "bigcode/starcoder"
-
-    @property
-    def prefix_token(self) -> str:
-        return self.end_of_text_token

helm/benchmark/window_services/t0pp_window_service.py

@@ -1,35 +0,0 @@
-from .encoder_decoder_window_service import EncoderDecoderWindowService
-from .tokenizer_service import TokenizerService
-
-
-class T0ppWindowService(EncoderDecoderWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length."""
-        # From https://arxiv.org/pdf/2110.08207.pdf, "we truncate input and target sequences to 1024 and 256 tokens,
-        # respectively. Following Raffel et al. (2020), we use packing to combine multiple training examples into
-        # a single sequence to reach the maximum sequence length."
-        return 1024
-
-    @property
-    def max_output_length(self) -> int:
-        return 256
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "</s>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "bigscience/T0pp"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token is the same as the end of text token."""
-        # echo=True is not supported
-        return ""

helm/benchmark/window_services/t511b_window_service.py

@@ -1,30 +0,0 @@
-from .encoder_decoder_window_service import EncoderDecoderWindowService
-from .tokenizer_service import TokenizerService
-
-
-class T511bWindowService(EncoderDecoderWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length."""
-        # From https://arxiv.org/pdf/1910.10683.pdf, "we use a maximum sequence length of 512".
-        # We subtract 1 to account for <extra_id_0> that gets appended to prompts.
-        return 512 - 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "</s>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "google/t5-11b"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token is the same as the end of text token."""
-        # echo=True is not supported
-        return ""