crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/vlm_run_specs.py

@@ -1,97 +0,0 @@
-from typing import List, Optional
-
-from .adaptation.adapter_spec import AdapterSpec
-from .adaptation.adapters.adapter_factory import ADAPT_GENERATION_MULTIMODAL
-from .metrics.metric import MetricSpec
-from .run_specs import run_spec_function, get_exact_match_metric_specs
-from .runner import RunSpec
-from .scenarios.scenario import ScenarioSpec
-
-
-############################################################
-# Prototypical adapter specs for VLM evaluation
-
-
-def get_vlm_generation_adapter_spec(
-    instructions: str = "",
-    input_prefix: str = "",
-    input_suffix: str = "",
-    output_prefix: str = "",
-    output_suffix: str = "",
-    max_tokens: int = 100,
-    stop_sequences: Optional[List[str]] = None,
-) -> AdapterSpec:
-    return AdapterSpec(
-        method=ADAPT_GENERATION_MULTIMODAL,
-        global_prefix="",
-        instructions=instructions,
-        input_prefix=input_prefix,
-        input_suffix=input_suffix,
-        output_prefix=output_prefix,
-        output_suffix=output_suffix,
-        instance_prefix="\n",
-        # We focus on zero-shot evaluation for now as most open VLMs only support a single image input
-        max_train_instances=0,
-        num_outputs=1,
-        max_tokens=max_tokens,
-        stop_sequences=stop_sequences if stop_sequences is not None else [],
-        random=None,
-    )
-
-
-############################################################
-# VHELM run specs
-
-
-@run_spec_function("viz_wiz")
-def get_viz_wiz_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.vision_language.viz_wiz_scenario.VizWizScenario", args={}
-    )
-
-    # TODO: finalize the adapter spec parameters once we add more models
-    adapter_spec: AdapterSpec = get_vlm_generation_adapter_spec(
-        input_prefix="User: ",
-        input_suffix="<end_of_utterance>",
-        output_prefix="\nAssistant: ",
-        output_suffix="<end_of_utterance>",
-        stop_sequences=["<end_of_utterance>"],
-    )
-
-    metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
-
-    run_spec_name: str = "viz_wiz"
-    return RunSpec(
-        name=run_spec_name,
-        scenario_spec=scenario_spec,
-        adapter_spec=adapter_spec,
-        metric_specs=metric_specs,
-        groups=[run_spec_name],
-    )
-
-
-@run_spec_function("vqa")
-def get_vqa_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.vision_language.vqa_scenario.VQAScenario", args={}
-    )
-
-    # TODO: finalize the adapter spec parameters once we add more models
-    adapter_spec: AdapterSpec = get_vlm_generation_adapter_spec(
-        input_prefix="User: ",
-        input_suffix="<end_of_utterance>",
-        output_prefix="\nAssistant: ",
-        output_suffix="<end_of_utterance>",
-        stop_sequences=["<end_of_utterance>"],
-    )
-
-    metric_specs: List[MetricSpec] = get_exact_match_metric_specs()
-
-    run_spec_name: str = "vqa"
-    return RunSpec(
-        name=run_spec_name,
-        scenario_spec=scenario_spec,
-        adapter_spec=adapter_spec,
-        metric_specs=metric_specs,
-        groups=[run_spec_name],
-    )

helm/benchmark/window_services/ai21_window_service.py

@@ -1,258 +0,0 @@
-import re
-
-from typing import List, Optional, Tuple
-from urllib.parse import unquote
-
-from helm.common.tokenization_request import (
-    TokenizationRequest,
-    TokenizationRequestResult,
-    TokenizationToken,
-    TextRange,
-)
-from .window_service import WindowService, EncodeResult
-from .tokenizer_service import TokenizerService
-from .gpt2_window_service import GPT2WindowService
-
-
-class AI21WindowService(WindowService):
-    """Tokenizes by making a request to the proxy server with REST endpoint: `/api/tokenize`."""
-
-    # AI21's tokenizer API rejects a tokenization request if the input sequence is too long, so
-    # we need to set an upper limit for the length of the request. Empirically, if the GPT2 tokenizer tokenizes a
-    # sequence to <= 11000 tokens, then it is most likely safe to assume that AI21's tokenization API will
-    # process this request.
-    MAX_TOKENIZATION_REQUEST_LENGTH: int = 11000
-
-    # The AI21 tokenizer throws the following error when sending a request with text that has too many characters:
-    # "Text must be under 100,000 characters (type=value_error)"
-    # Sending a request with 100,000 characters seem to work though.
-    MAX_CHARACTER_LENGTH: int = 100_000
-
-    NOT_IMPLEMENTED_ERROR_MESSAGE: str = (
-        "AI21 only gave API access to their tokenizer, so this method is not supported."
-    )
-
-    def __init__(self, service: TokenizerService, gpt2_window_service: GPT2WindowService):
-        # We need the `TokenizerService` to make requests to the server.
-        self.service: TokenizerService = service
-        # As explained above, we need a `GPT2WindowService` to help tokenize long text sequences.
-        self.gpt2_window_service: GPT2WindowService = gpt2_window_service
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "ai21/j1"
-
-    @property
-    def max_sequence_length(self) -> int:
-        """
-        The max token length of the model in. The AI21 server automatically prepends a token to every prompt,
-        so the actual max sequence length is 2048-1 = 2047.
-        """
-        return 2047
-
-    @property
-    def max_request_length(self) -> int:
-        """The max sequence length is the same as the max request length for AI21."""
-        return self.max_sequence_length
-
-    @property
-    def end_of_text_token(self) -> str:
-        # TODO: I'm not sure what their end of text token is. I don't think it's documented.
-        return " "
-
-    @property
-    def prefix_token(self) -> str:
-        """AI21 tokenizers do no have a prefix token"""
-        return ""
-
-    def encode(self, text: str, truncation: bool = False, max_length: Optional[int] = None) -> EncodeResult:
-        """
-        Encodes the input text to tokens.
-        """
-        tokens: List[TokenizationToken]
-        normalized_text: str
-        tokens, normalized_text = self._make_long_tokenization_request(text)
-        # The end position of the last token should be the end of the text.
-        if len(tokens) > 0:
-            assert tokens[-1].text_range is not None
-            assert tokens[-1].text_range.end == len(normalized_text)
-
-        return EncodeResult(text=normalized_text, tokens=tokens)
-
-    def decode(self, tokens: List[TokenizationToken], normalized_text: Optional[str] = None) -> str:
-        """
-        Given the model and a list of tokens, outputs the corresponding text.
-
-        For models using the GPT-2 tokenizer, the tokens are integers; for AI21
-        models, the tokens are `TokenizationToken`s.
-
-        Some tokenizers (e.g. AI21) normalize the text before encoding it and
-        thus require the `normalized_text` for decoding.
-        """
-        if not tokens:
-            return ""
-
-        # `normalized_text` is necessary for decoding AI21 tokens.
-        assert normalized_text, "The AI21 tokenizer needs `normalized_text` for decoding"
-        for j in range(len(tokens) - 1):
-            first_text_range = tokens[j].text_range
-            second_text_range = tokens[j + 1].text_range
-            assert first_text_range is not None
-            assert second_text_range is not None
-            assert (
-                first_text_range.end == second_text_range.start
-            ), "The tokens to be decoded must form a substring of `normalized_text`."
-
-        token_texts: List[str] = []
-        # The format of AI21 byte token representations. e.g. <0xE8>
-        byte_pattern = "<0x[0-9A-F]{2}>"
-        i: int = 0
-        while i < len(tokens):
-            # If there are byte tokens, aggregates them to a string
-            token_value = tokens[i].value
-            assert isinstance(token_value, str)
-            if re.match(byte_pattern, token_value):
-                bytestring = ""
-                while i < len(tokens) and re.match(byte_pattern, token_value):
-                    # e.g. <0xE8> -> \xE8
-                    bytestring += "\\" + token_value[2:-1]
-                    i += 1
-                # Convert to encoded URI (e.g., %e2%80%99) and decode
-                token_text = unquote(bytestring.replace("\\x", "%"))
-            # Not a byte token: retrieves the token text based on text_range.
-            else:
-                token: TokenizationToken = tokens[i]
-                assert token.text_range is not None
-                token_text = normalized_text[token.text_range.start : token.text_range.end]
-                i += 1
-            token_texts.append(token_text)
-        return "".join(token_texts)
-
-    def tokenize(self, text: str) -> List[str]:
-        """
-        Tokenizes the text via the /api/tokenize REST endpoint.
-        """
-        response: TokenizationRequestResult = self._make_tokenization_request(text)
-        result = []
-        for token in response.tokens:
-            assert isinstance(token.value, str)
-            result.append(token.value)
-        return result
-
-    def get_num_tokens(self, text: str) -> int:
-        """Tokenizes the text using the GPT-2 tokenizer and returns the number of tokens."""
-        return len(self.tokenize(text))
-
-    def fits_within_context_window(self, text: str, expected_completion_token_length: int = 0) -> bool:
-        return (
-            len(text) <= AI21WindowService.MAX_CHARACTER_LENGTH
-            and self.get_num_tokens(text) + expected_completion_token_length <= self.max_request_length
-        )
-
-    def truncate_from_right(self, text: str, expected_completion_token_length: int = 0) -> str:
-        """
-        Truncates the text using the AI21 Jurassic tokenizer.
-        First, ensures the text is shorter than `AI21Tokenizer.MAX_CHARACTER_LENGTH` long.
-        Tokenizes, then truncates the list of tokens to fit within the context window minus the
-        expected completion length (defaults to 0), then uses the start of the text range of the first
-        token and the end of the text range of the last token of the truncated list of tokens to
-        build the truncated text.
-        """
-        text = text[: AI21WindowService.MAX_CHARACTER_LENGTH]
-        response: TokenizationRequestResult = self._make_tokenization_request(text)
-
-        # Only look at the first `self.max_request_length` - `expected_completion_token_length`
-        # number of tokens to the fit the text within the context window.
-        # Each token is represented like this: {'text': '▁Hello', 'textRange': {'start': 0, 'end': 5}}
-        max_length: int = self.max_request_length - expected_completion_token_length
-        tokens: List[TokenizationToken] = response.tokens[:max_length]
-
-        # If there is no tokens, just return the original text
-        if len(tokens) == 0:
-            return text
-
-        # AI21 uses "_" to represent a single space in their tokens, so we have to build the new text from the
-        # original text after truncation using the text ranges of tokens generated from the original text.
-        assert tokens[0].text_range is not None
-        first_text_range: TextRange = tokens[0].text_range
-        assert tokens[-1].text_range is not None
-        last_text_range: TextRange = tokens[-1].text_range
-        start: int = first_text_range.start
-        end: int = last_text_range.end
-        truncated_text: str = text[start:end]
-
-        # HACK: For the vast majority of cases, the above logic works, but there are a few where the
-        # token count exceeds `max_length` by 1. This might be a bug with the AI21 tokenizer API.
-        # We handle those by removing characters one by one until it fits within the context window.
-        while not self.fits_within_context_window(truncated_text, expected_completion_token_length):
-            end -= 1
-            truncated_text = text[start:end]
-        return truncated_text
-
-    def _make_tokenization_request(self, text: str) -> TokenizationRequestResult:
-        """Sends a request to the server to tokenize the text via the `TokenizerService`."""
-        return self.service.tokenize(TokenizationRequest(text=text, tokenizer=self.tokenizer_name))
-
-    def _make_long_tokenization_request(self, text: str) -> Tuple[List[TokenizationToken], str]:
-        """If the text is too long  (longer than 11,000 tokens when tokenized by the GPT-2 tokenizer),
-        the AI21 server will close the connection. Therefore, we need to split the text into smaller
-        chunks, tokenize each chunk, and re-assemble the tokenization results."""
-        # Uses the number of gpt2-style tokens as a measure of text length.
-        gpt2_tokens: List[TokenizationToken] = self.gpt2_window_service.encode(text).tokens
-
-        # If the text is short, just makes one request and returns the result.
-        if len(gpt2_tokens) < AI21WindowService.MAX_TOKENIZATION_REQUEST_LENGTH:
-            result: TokenizationRequestResult = self._make_tokenization_request(text)
-            return result.tokens, result.text
-        # Otherwise, splits the text to chunks, tokenizes each chunk, and re-assembles them.
-        else:
-            all_tokens: List[TokenizationToken] = []
-            normalized_text_chunks: List[str] = []
-            # The number of gpt2-style tokens we have tokenized with the AI21 tokenizer.
-            num_processed_tokens: int = 0
-            # The length of the (normalized) text string we have tokenized with the AI21 tokenizer.
-            num_processed_positions: int = 0
-            while num_processed_tokens < len(gpt2_tokens):
-                token_chunk_size: int = min(
-                    len(gpt2_tokens) - num_processed_tokens, AI21WindowService.MAX_TOKENIZATION_REQUEST_LENGTH
-                )
-                token_chunk: List[TokenizationToken] = gpt2_tokens[
-                    num_processed_tokens : num_processed_tokens + token_chunk_size
-                ]
-                text_chunk: str = self.gpt2_window_service.decode(token_chunk)
-                # We need to avoid generating byte tokens when splitting the text
-                while text_chunk.endswith("\ufffd"):
-                    token_chunk_size -= 1
-                    token_chunk = gpt2_tokens[num_processed_tokens : num_processed_tokens + token_chunk_size]
-                    text_chunk = self.gpt2_window_service.decode(token_chunk)
-                chunk_result: TokenizationRequestResult = self._make_tokenization_request(text_chunk)
-                chunk_tokens: List[TokenizationToken]
-                normalized_text_chunk: str
-                chunk_tokens, normalized_text_chunk = chunk_result.tokens, chunk_result.text
-                # Removes the empty tokens introduced by the split.
-                assert chunk_tokens[0].text_range is not None
-                if num_processed_tokens != 0 and chunk_tokens[0].text_range.start == chunk_tokens[0].text_range.end:
-                    chunk_tokens = chunk_tokens[1:]
-                else:
-                    chunk_tokens = chunk_tokens[:]
-
-                # Shifts the start and end index of each token
-                shifted_tokens: List[TokenizationToken] = []
-                for token in chunk_tokens:
-                    assert token.text_range is not None
-                    shifted_tokens.append(
-                        TokenizationToken(
-                            value=token.value,
-                            text_range=TextRange(
-                                start=token.text_range.start + num_processed_positions,
-                                end=token.text_range.end + num_processed_positions,
-                            ),
-                        )
-                    )
-                all_tokens.extend(shifted_tokens)
-                normalized_text_chunks.append(normalized_text_chunk)
-                num_processed_tokens += token_chunk_size
-                num_processed_positions += len(normalized_text_chunk)
-
-            return all_tokens, "".join(normalized_text_chunks)

helm/benchmark/window_services/cohere_window_service.py

@@ -1,163 +0,0 @@
-from typing import List, Optional
-
-from helm.proxy.tokenizers.cohere_tokenizer import CohereTokenizer
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-from .window_service import EncodeResult
-from helm.common.tokenization_request import (
-    TokenizationRequest,
-    TokenizationRequestResult,
-    TokenizationToken,
-)
-
-
-class CohereWindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def tokenizer_name(self) -> str:
-        return "cohere/cohere"
-
-    @property
-    def max_sequence_length(self) -> int:
-        """
-        The max length of the model input. Similar to MT-NLG, Cohere does not predict the logprob of
-        the first input token so `max_sequence_length` is one token shorter than `max_request_length`.
-        """
-        return self.max_request_length - 1
-
-    @property
-    def max_request_length(self) -> int:
-        """
-        The max request length of the model. For Cohere, this is the same as the `max_sequence_length`.
-        If we exceed the `max_sequence_length`, we get the following error:
-
-        Request failed with too many tokens: total number of tokens (prompt and prediction) cannot
-        exceed 2048 - received 2049. Try using a shorter prompt or a smaller max_tokens value.
-        """
-        return 2048
-
-    @property
-    def end_of_text_token(self) -> str:
-        """
-        The end of text token. Cohere does not have one.
-        """
-        return ""
-
-    @property
-    def prefix_token(self) -> str:
-        """
-        The prefix token. Cohere does not return the log prob for the first token when `echo_prompt` is True.
-        """
-        # Cohere recommended ":", but we can try out different values
-        return ":"
-
-    def encode(self, text: str, truncation: bool = False, max_length: Optional[int] = None) -> EncodeResult:
-        """
-        Encodes the input text to tokens.
-        """
-        if max_length is None:
-            max_length = self.max_request_length
-
-        response: TokenizationRequestResult
-        tokens: List[TokenizationToken] = []
-        if truncation or len(text) <= CohereTokenizer.TOKENIZE_API_MAX_TEXT_LENGTH:
-            response = self.service.tokenize(
-                TokenizationRequest(
-                    text,
-                    tokenizer=self.tokenizer_name,
-                    # The Cohere API does not support decoding, so set `encode` to False to get the value of tokens
-                    # as strings so we can simply concatenate them when we need to decode.
-                    encode=False,
-                    truncation=truncation,
-                    max_length=max_length,
-                )
-            )
-            tokens = response.tokens
-        else:
-            # Perform chunk encoding: Cohere doesn't support long sequences, so break it up into chunks
-            # and make a request for each chunk.
-            # This can potentially break up valid tokens at the end of the chunk, but the chunk size
-            # is large enough that this happens infrequently.
-            chunk_size: int = CohereTokenizer.TOKENIZE_API_MAX_TEXT_LENGTH
-            for i in range(0, len(text), chunk_size):
-                chunk: str = text[i : chunk_size + i]
-                response = self.service.tokenize(
-                    TokenizationRequest(chunk, tokenizer=self.tokenizer_name, encode=False, truncation=False)
-                )
-                tokens.extend(response.tokens)
-
-        return EncodeResult(text=text, tokens=tokens)
-
-    def get_num_tokens(self, text: str) -> int:
-        """Tokenizes the text and returns the number of tokens."""
-        # We need this check since we can't pass in empty string via the `tokenize` endpoint
-        if len(text) == 0:
-            return 0
-        return len(self.encode(text).tokens)
-
-    def decode(self, tokens: List[TokenizationToken], normalized_text: Optional[str] = None) -> str:
-        """
-        The Cohere API does not support decoding, but we're able to recover the original text from the
-        values of the tokens by concatenating them.
-
-        Note this logic currently only works with English text.
-        """
-        token_strings = []
-        for token in tokens:
-            assert isinstance(token.value, str)
-            token_strings.append(token.value)
-        return "".join(token_strings)
-
-    def fits_within_context_window(self, text: str, expected_completion_token_length: int = 0) -> bool:
-        """
-        Checks if the given text fits within the context window given by `max_request_length`
-        taking to account the expected completion length (defaults to 0).
-
-        According to https://docs.cohere.ai/tokenize-reference#request, for tokenize, text: "the string to
-        be tokenized, the minimum text length is 1 character, and the maximum text length is 65,536 characters.",
-        so first check if the text has fewer than 65,536 characters.
-        """
-        return (
-            len(text) <= CohereTokenizer.TOKENIZE_API_MAX_TEXT_LENGTH
-            and self.get_num_tokens(text) + expected_completion_token_length <= self.max_request_length
-        )
-
-    def truncate_from_right(self, text: str, expected_completion_token_length: int = 0) -> str:
-        """
-        Truncates text from the right to fit within the context window given by `max_request_length`
-        minus the expected completion length (defaults to 0).
-        """
-        # First truncate the text so it's within `CohereClient.TOKENIZE_MAX_TEXT_LENGTH` length.
-        text = text[: CohereTokenizer.TOKENIZE_API_MAX_TEXT_LENGTH]
-
-        max_length: int = self.max_request_length - expected_completion_token_length
-        result: str = self.decode(self.encode(text, truncation=True, max_length=max_length).tokens)
-
-        # HACK: For the vast majority of cases, the above logic works, but it sometimes doesn't work
-        # for non-English text, since Cohere technically only supports English at the moment.
-        while not self.fits_within_context_window(result, expected_completion_token_length):
-            result = result[:-1]
-
-        return result
-
-
-class CohereCommandWindowService(CohereWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_request_length(self) -> int:
-        """
-        The max request length of the model. For Cohere, this is the same as the `max_sequence_length`.
-        If we exceed the `max_sequence_length`, we get the following error:
-
-        Request failed with too many tokens: total number of tokens (prompt and prediction) cannot
-        exceed 2048 - received 2049. Try using a shorter prompt or a smaller max_tokens value.
-
-        For the Command model, in rare situations, the co.tokenize returns a shorter list of tokens
-        than the co.generate. This causes sequence length errors for rare inputs. Cohere's advice is
-        to reduce the sequence length to 2020 to avoid these issues.
-        """
-        return 2020

helm/benchmark/window_services/flan_t5_window_service.py

@@ -1,29 +0,0 @@
-from .encoder_decoder_window_service import EncoderDecoderWindowService
-from .tokenizer_service import TokenizerService
-
-
-class FlanT5WindowService(EncoderDecoderWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length."""
-        # We subtract 1 to account for <extra_id_0> that gets appended to prompts.
-        return 512 - 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "</s>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "google/flan-t5-xxl"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token is the same as the end of text token."""
-        # echo=True is not supported
-        return ""

helm/benchmark/window_services/gpt2_window_service.py

@@ -1,32 +0,0 @@
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class GPT2WindowService(LocalWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length of this tokenizer."""
-        return 1024
-
-    @property
-    def max_request_length(self) -> int:
-        """Return the max request length of GPT-2."""
-        return self.max_sequence_length + 1
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "<|endoftext|>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "huggingface/gpt2"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token for models that uses the GPT-2 tokenizer is the end of text token."""
-        return self.end_of_text_token

helm/benchmark/window_services/huggingface_window_service.py

@@ -1,60 +0,0 @@
-from typing import Optional
-from helm.proxy.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
-from .local_window_service import LocalWindowService
-from .tokenizer_service import TokenizerService
-
-
-class HuggingFaceWindowService(LocalWindowService):
-    def __init__(
-        self,
-        service: TokenizerService,
-        tokenizer_name: str,
-        pretrained_model_name_or_path: Optional[str] = None,
-        max_sequence_length: Optional[int] = None,
-        max_request_length: Optional[int] = None,
-        end_of_text_token: Optional[str] = None,
-        prefix_token: Optional[str] = None,
-        **kwargs
-    ):
-        super().__init__(service)
-        self._tokenizer_name = tokenizer_name
-        # Override max_sequence_length, max_request_length, end_of_text_token
-        # and prefix_token if provided as an argument.
-        # Otherwise, auto-infer them from the Hugging Face tokenizer.
-        #
-        # Note that many Hugging Face tokenizers have incorrect sequence lengths,
-        # so it is recommended to set this manually.
-        with HuggingFaceTokenizer.get_tokenizer(
-            helm_tokenizer_name=tokenizer_name,
-            pretrained_model_name_or_path=pretrained_model_name_or_path or tokenizer_name,
-            **kwargs,
-        ) as tokenizer:
-            self._max_sequence_length = max_sequence_length or tokenizer.model_max_length
-            self._max_request_length = max_request_length or self._max_sequence_length
-            self._end_of_text_token = end_of_text_token or tokenizer.eos_token or ""
-            self._prefix_token = prefix_token or tokenizer.bos_token or ""
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return self._tokenizer_name
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length of this tokenizer."""
-        return self._max_sequence_length
-
-    @property
-    def max_request_length(self) -> int:
-        """Return the max request length of this tokenizer."""
-        return self._max_request_length
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return self._end_of_text_token
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token."""
-        return self._prefix_token

helm/benchmark/window_services/t0pp_window_service.py

@@ -1,35 +0,0 @@
-from .encoder_decoder_window_service import EncoderDecoderWindowService
-from .tokenizer_service import TokenizerService
-
-
-class T0ppWindowService(EncoderDecoderWindowService):
-    def __init__(self, service: TokenizerService):
-        super().__init__(service)
-
-    @property
-    def max_sequence_length(self) -> int:
-        """Return the max sequence length."""
-        # From https://arxiv.org/pdf/2110.08207.pdf, "we truncate input and target sequences to 1024 and 256 tokens,
-        # respectively. Following Raffel et al. (2020), we use packing to combine multiple training examples into
-        # a single sequence to reach the maximum sequence length."
-        return 1024
-
-    @property
-    def max_output_length(self) -> int:
-        return 256
-
-    @property
-    def end_of_text_token(self) -> str:
-        """The end of text token."""
-        return "</s>"
-
-    @property
-    def tokenizer_name(self) -> str:
-        """Name of the tokenizer to use when sending a request."""
-        return "bigscience/T0pp"
-
-    @property
-    def prefix_token(self) -> str:
-        """The prefix token is the same as the end of text token."""
-        # echo=True is not supported
-        return ""