crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/proxy/services/test_service.py

@@ -3,6 +3,7 @@ import pytest
 import shutil
 import tempfile
 
+from helm.benchmark.model_deployment_registry import ModelDeployment, get_model_deployment
 from helm.common.authentication import Authentication
 from helm.common.request import Request
 from helm.proxy.accounts import AuthenticationError, Accounts
@@ -34,7 +35,9 @@ class TestServerService:
 
     def test_make_request(self):
         num_completions = 2
-        request = Request(prompt="1 2 3", model="simple/model1", num_completions=num_completions)
+        request = Request(
+            prompt="1 2 3", model="simple/model1", model_deployment="simple/model1", num_completions=num_completions
+        )
         result = self.service.make_request(self.auth, request)
         assert len(result.completions) == num_completions
 
@@ -194,24 +197,12 @@ def helper_prod_test_service(request: Request, expected_text: str):
         # Consistency of log probs
         assert completion.logprob == sum(token.logprob for token in completion.tokens)
 
-        for token in completion.tokens[1:]:
-            assert len(token.top_logprobs) == request.top_k_per_token
-
-            # If generated token was one of the top, make sure has the right probability
-            if token.text in token.top_logprobs:
-                assert token.logprob == token.top_logprobs[token.text]
-
-            # If temperature = 0, then make sure we're getting the top probability token
-            if request.temperature == 0:
-                assert token.text in token.top_logprobs
-                assert token.logprob == max(token.top_logprobs.values())
-
     # Make sure we get the expected_text in one of the completions
     assert any(completion.text == expected_text for completion in result.completions)
 
 
 # Models that we want to test
-prod_models = ["openai/davinci", "ai21/j1-jumbo"]
+prod_model_deployments = ["openai/davinci", "ai21/j1-jumbo"]
 
 
 # TODO: put a flag on this so that it's easy to use pytest to still run these slow tests
@@ -220,8 +211,17 @@ prod_models = ["openai/davinci", "ai21/j1-jumbo"]
 def test_prod_continue():
     # Test that we're continuing
     prompt = "Paris is the capital of"
-    for model in prod_models:
-        request = Request(prompt=prompt, model=model, max_tokens=1, num_completions=1, temperature=0)
+    for model_deployment_name in prod_model_deployments:
+        model_deployment: ModelDeployment = get_model_deployment(model_deployment_name)
+        model_name: str = model_deployment.model_name or model_deployment.name
+        request = Request(
+            prompt=prompt,
+            model=model_name,
+            model_deployment=model_deployment_name,
+            max_tokens=1,
+            num_completions=1,
+            temperature=0,
+        )
         helper_prod_test_service(request, " France")
 
 
@@ -229,6 +229,15 @@ def test_prod_continue():
 def test_prod_echo():
     # If we're echoing the prompt, make sure we're getting the same thing back
     prompt = "I like pickles."
-    for model in prod_models:
-        request = Request(prompt=prompt, model=model, max_tokens=0, num_completions=1, echo_prompt=True)
+    for model_deployment_name in prod_model_deployments:
+        model_deployment: ModelDeployment = get_model_deployment(model_deployment_name)
+        model_name: str = model_deployment.model_name or model_deployment.name
+        request = Request(
+            prompt=prompt,
+            model=model_name,
+            model_deployment=model_deployment_name,
+            max_tokens=0,
+            num_completions=1,
+            echo_prompt=True,
+        )
         helper_prod_test_service(request, prompt)

helm/proxy/test_accounts.py

@@ -0,0 +1,32 @@
+import os
+import pytest
+import tempfile
+
+from helm.proxy.accounts import Accounts, Authentication, InsufficientQuotaError, Usage
+
+
+class TestAutoTokenCounter:
+    def setup_method(self, method):
+        accounts_file = tempfile.NamedTemporaryFile(delete=False)
+        self.accounts_path: str = accounts_file.name
+        self.accounts = Accounts(self.accounts_path)
+        self.root_auth = Authentication(Accounts.DEFAULT_API_KEY)
+
+    def teardown_method(self, method):
+        os.remove(self.accounts_path)
+
+    def test_check_can_use(self):
+        model_group = "anthropic"
+        account = self.accounts.create_account(self.root_auth)
+
+        # Cannot use this account because no quota was added
+        with pytest.raises(InsufficientQuotaError):
+            self.accounts.check_can_use(account.api_key, model_group)
+
+        # Add monthly quota
+        account.usages[model_group] = {}
+        account.usages[model_group]["monthly"] = Usage(quota=1000)
+        self.accounts.update_account(self.root_auth, account)
+
+        # Now this account has quota and can be used
+        self.accounts.check_can_use(account.api_key, model_group)

helm/proxy/token_counters/auto_token_counter.py

@@ -1,42 +1,42 @@
-from typing import Dict, List
-
-from helm.common.request import Request, Sequence
-from helm.proxy.clients.huggingface_client import HuggingFaceClient
-from .ai21_token_counter import AI21TokenCounter
-from .cohere_token_counter import CohereTokenCounter
-from .free_token_counter import FreeTokenCounter
-from .gooseai_token_counter import GooseAITokenCounter
-from .openai_token_counter import OpenAITokenCounter
+from typing import List
+from helm.benchmark.model_deployment_registry import ModelDeployment, get_model_deployment
+
+from helm.common.request import Request, GeneratedOutput
+from helm.tokenizers.auto_tokenizer import AutoTokenizer
+from helm.common.tokenization_request import TokenizationRequest, TokenizationRequestResult
 from .token_counter import TokenCounter
 
 
 class AutoTokenCounter(TokenCounter):
-    """Automatically count tokens based on the organization."""
-
-    def __init__(self, huggingface_client: HuggingFaceClient):
-        self.token_counters: Dict[str, TokenCounter] = {}
-        self.huggingface_client: HuggingFaceClient = huggingface_client
-
-    def get_token_counter(self, organization: str) -> TokenCounter:
-        """Return a token counter based on the organization."""
-        token_counter = self.token_counters.get(organization)
-        if token_counter is None:
-            if organization == "openai":
-                token_counter = OpenAITokenCounter(self.huggingface_client)
-            elif organization == "ai21":
-                token_counter = AI21TokenCounter()
-            elif organization == "gooseai":
-                token_counter = GooseAITokenCounter()
-            elif organization == "cohere":
-                token_counter = CohereTokenCounter()
+    """Automatically count tokens based on the model_deployment."""
+
+    def __init__(self, auto_tokenizer: AutoTokenizer):
+        self.auto_tokenizer: AutoTokenizer = auto_tokenizer
+
+    def count_tokens(self, request: Request, completions: List[GeneratedOutput]) -> int:
+        """Counts tokens based on the model deployment.
+
+        This counts the number of tokens in the request and completions.
+        Both input and output tokens are counted. For some model providers,
+        this method will return a larger number of tokens than the actual
+        token count used for billing. For example, GooseAI only charges for
+        (output_tokens - 25) rather than (input_tokens + output_tokens)."""
+        model_deployment: ModelDeployment = get_model_deployment(request.model_deployment)
+        assert model_deployment.tokenizer_name
+        tokenizer_name = model_deployment.tokenizer_name
+
+        num_completion_tokens = 0
+        for completion in completions:
+            if completion.tokens:
+                num_completion_tokens += len(completion.tokens)
             else:
-                token_counter = FreeTokenCounter()
-            self.token_counters[organization] = token_counter
-        return token_counter
-
-    def count_tokens(self, request: Request, completions: List[Sequence]) -> int:
-        """
-        Counts tokens based on the organization.
-        """
-        token_counter: TokenCounter = self.get_token_counter(request.model_organization)
-        return token_counter.count_tokens(request, completions)
+                tokenized_completion: TokenizationRequestResult = self.auto_tokenizer.tokenize(
+                    TokenizationRequest(request.prompt, tokenizer=tokenizer_name)
+                )
+                num_completion_tokens += len(tokenized_completion.tokens)
+
+        tokenized_prompt: TokenizationRequestResult = self.auto_tokenizer.tokenize(
+            TokenizationRequest(request.prompt, tokenizer=tokenizer_name)
+        )
+        num_prompt_tokens = len(tokenized_prompt.tokens)
+        return num_prompt_tokens + num_completion_tokens

helm/proxy/token_counters/test_auto_token_counter.py

@@ -0,0 +1,164 @@
+from typing import List
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
+
+from helm.common.request import Request, GeneratedOutput, Token
+from helm.tokenizers.auto_tokenizer import AutoTokenizer
+from helm.proxy.token_counters.auto_token_counter import AutoTokenCounter
+
+
+class TestAutoTokenCounter:
+    def test_count_tokens_openai(self):
+        token_counter = AutoTokenCounter(
+            AutoTokenizer(credentials={}, cache_backend_config=BlackHoleCacheBackendConfig())
+        )
+        # The following prompt has 51 tokens according to the GPT-2 tokenizer
+        request = Request(
+            model="openai/text-davinci-002",
+            model_deployment="openai/text-davinci-002",
+            prompt="The Center for Research on Foundation Models (CRFM) is "
+            "an interdisciplinary initiative born out of the Stanford "
+            "Institute for Human-Centered Artificial Intelligence (HAI) "
+            "that aims to make fundamental advances in the study, development, "
+            "and deployment of foundation models.",
+        )
+        completions: List[GeneratedOutput] = [
+            GeneratedOutput(
+                text=" The CRFM is dedicated to advancing our knowledge of the foundations of artificial intelligence "
+                "(AI) and related fields. It focuses on foundational questions in AI, which are",
+                logprob=-49.00783279519999,
+                tokens=[
+                    Token(text=" The", logprob=-1.8096403),
+                    Token(text=" CR", logprob=-1.2861944),
+                    Token(text="FM", logprob=-0.0032369632),
+                    Token(text=" is", logprob=-1.4355252),
+                    Token(text=" dedicated", logprob=-3.814422),
+                    Token(text=" to", logprob=-0.009623392),
+                    Token(text=" advancing", logprob=-2.6732886),
+                    Token(text=" our", logprob=-3.123714),
+                    Token(text=" knowledge", logprob=-3.030337),
+                    Token(text=" of", logprob=-0.46280858),
+                    Token(text=" the", logprob=-1.4058315),
+                    Token(text=" foundations", logprob=-2.0638132),
+                    Token(text=" of", logprob=-0.2607486),
+                    Token(text=" artificial", logprob=-1.1653417),
+                    Token(text=" intelligence", logprob=-0.03756146),
+                    Token(text=" (", logprob=-2.019812),
+                    Token(text="AI", logprob=-0.03869382),
+                    Token(text=")", logprob=-0.49895737),
+                    Token(text=" and", logprob=-0.81909865),
+                    Token(text=" related", logprob=-2.611718),
+                    Token(text=" fields", logprob=-0.7640527),
+                    Token(text=".", logprob=-1.8066244),
+                    Token(text=" It", logprob=-2.2856107),
+                    Token(text=" focuses", logprob=-3.3735154),
+                    Token(text=" on", logprob=-0.13244776),
+                    Token(text=" foundational", logprob=-1.2640914),
+                    Token(text=" questions", logprob=-2.010647),
+                    Token(text=" in", logprob=-1.980726),
+                    Token(text=" AI", logprob=-0.5709368),
+                    Token(text=",", logprob=-1.036094),
+                    Token(text=" which", logprob=-3.826836),
+                    Token(text=" are", logprob=-1.3858839),
+                ],
+            )
+        ]
+
+        # Verified against https://beta.openai.com/tokenizer. Prompt + completions = 51 + 32.
+        assert token_counter.count_tokens(request, completions) == 51 + 32
+
+    def test_count_tokens_anthropic(self):
+        token_counter = AutoTokenCounter(
+            AutoTokenizer(credentials={}, cache_backend_config=BlackHoleCacheBackendConfig())
+        )
+        request = Request(
+            model="anthropic/claude-instant-v1",
+            model_deployment="anthropic/claude-instant-v1",
+            prompt="\n\nHuman:The Center for Research on Foundation Models (CRFM) is "
+            "an interdisciplinary initiative born out of the Stanford "
+            "Institute for Human-Centered Artificial Intelligence (HAI) "
+            "that aims to make fundamental advances in the study, development, "
+            "and deployment of foundation models.\n\nAssistant:",
+        )
+        completions: List[GeneratedOutput] = [
+            GeneratedOutput(
+                text="Thank you for the background information. The Center for Research "
+                "on Foundation Models sounds like an interesting initiative focused on "
+                "advancing research and responsible development of large AI models. I "
+                "don't have any personal thoughts on it, but I'm happy to discuss or "
+                "provide more information if helpful. As an AI assistant, I don't have "
+                "subjective opinions.",
+                logprob=0,
+                tokens=[
+                    Token(text="Thank", logprob=0),
+                    Token(text=" you", logprob=0),
+                    Token(text=" for", logprob=0),
+                    Token(text=" the", logprob=0),
+                    Token(text=" background", logprob=0),
+                    Token(text=" information", logprob=0),
+                    Token(text=".", logprob=0),
+                    Token(text=" The", logprob=0),
+                    Token(text=" Center", logprob=0),
+                    Token(text=" for", logprob=0),
+                    Token(text=" Research", logprob=0),
+                    Token(text=" on", logprob=0),
+                    Token(text=" Foundation", logprob=0),
+                    Token(text=" Models", logprob=0),
+                    Token(text=" sounds", logprob=0),
+                    Token(text=" like", logprob=0),
+                    Token(text=" an", logprob=0),
+                    Token(text=" interesting", logprob=0),
+                    Token(text=" initiative", logprob=0),
+                    Token(text=" focused", logprob=0),
+                    Token(text=" on", logprob=0),
+                    Token(text=" advancing", logprob=0),
+                    Token(text=" research", logprob=0),
+                    Token(text=" and", logprob=0),
+                    Token(text=" responsible", logprob=0),
+                    Token(text=" development", logprob=0),
+                    Token(text=" of", logprob=0),
+                    Token(text=" large", logprob=0),
+                    Token(text=" AI", logprob=0),
+                    Token(text=" models", logprob=0),
+                    Token(text=".", logprob=0),
+                    Token(text=" I", logprob=0),
+                    Token(text=" don", logprob=0),
+                    Token(text="'t", logprob=0),
+                    Token(text=" have", logprob=0),
+                    Token(text=" any", logprob=0),
+                    Token(text=" personal", logprob=0),
+                    Token(text=" thoughts", logprob=0),
+                    Token(text=" on", logprob=0),
+                    Token(text=" it", logprob=0),
+                    Token(text=",", logprob=0),
+                    Token(text=" but", logprob=0),
+                    Token(text=" I", logprob=0),
+                    Token(text="'m", logprob=0),
+                    Token(text=" happy", logprob=0),
+                    Token(text=" to", logprob=0),
+                    Token(text=" discuss", logprob=0),
+                    Token(text=" or", logprob=0),
+                    Token(text=" provide", logprob=0),
+                    Token(text=" more", logprob=0),
+                    Token(text=" information", logprob=0),
+                    Token(text=" if", logprob=0),
+                    Token(text=" helpful", logprob=0),
+                    Token(text=".", logprob=0),
+                    Token(text=" As", logprob=0),
+                    Token(text=" an", logprob=0),
+                    Token(text=" AI", logprob=0),
+                    Token(text=" assistant", logprob=0),
+                    Token(text=",", logprob=0),
+                    Token(text=" I", logprob=0),
+                    Token(text=" don", logprob=0),
+                    Token(text="'t", logprob=0),
+                    Token(text=" have", logprob=0),
+                    Token(text=" subjective", logprob=0),
+                    Token(text=" opinions", logprob=0),
+                    Token(text=".", logprob=0),
+                ],
+                finish_reason=None,
+                multimodal_content=None,
+            )
+        ]
+
+        assert token_counter.count_tokens(request, completions) == 126

helm/proxy/token_counters/token_counter.py

@@ -1,15 +1,13 @@
 from abc import ABC, abstractmethod
 from typing import List
 
-from helm.common.request import Sequence, Request
+from helm.common.request import GeneratedOutput, Request
 
 
 class TokenCounter(ABC):
     """Counts the number of tokens used given `Request` and completions."""
 
     @abstractmethod
-    def count_tokens(self, request: Request, completions: List[Sequence]) -> int:
-        """
-        Counts the total number of tokens given a request and completions.
-        """
+    def count_tokens(self, request: Request, completions: List[GeneratedOutput]) -> int:
+        """Counts the total number of tokens given a request and completions."""
         pass

helm/{proxy/tokenizers → tokenizers}/ai21_tokenizer.py

@@ -1,4 +1,4 @@
-from typing import Dict, List
+from typing import Any, Dict, List
 import requests
 
 from dacite import from_dict
@@ -12,7 +12,7 @@ from helm.common.tokenization_request import (
     DecodeRequest,
     DecodeRequestResult,
 )
-from helm.proxy.clients.ai21_utils import AI21RequestError, handle_failed_request
+from helm.clients.ai21_utils import AI21RequestError, handle_failed_request
 from .tokenizer import Tokenizer
 
 
@@ -28,7 +28,7 @@ class AI21Tokenizer(Tokenizer):
         # TODO: Does not support encoding
         raw_request: Dict[str, str] = {"text": request.text}
 
-        def do_it():
+        def do_it() -> Dict[str, Any]:
             response = requests.post(
                 "https://api.ai21.com/studio/v1/tokenize",
                 headers={"Authorization": f"Bearer {self.api_key}"},

helm/{proxy/tokenizers → tokenizers}/aleph_alpha_tokenizer.py

@@ -31,7 +31,7 @@ class AlephAlphaTokenizer(CachingTokenizer):
     def __init__(self, api_key: str, cache_config: CacheConfig) -> None:
         super().__init__(cache_config)
         self.api_key: str = api_key
-        self._aleph_alpha_client = AlephAlphaPythonClient(token=api_key)
+        self._aleph_alpha_client = AlephAlphaPythonClient(token=api_key) if api_key else None
         self._tokenizer_name_to_tokenizer: Dict[str, InternalTokenizer] = {}
 
     def _get_tokenizer(self, tokenizer_name: str) -> InternalTokenizer:
@@ -40,6 +40,8 @@ class AlephAlphaTokenizer(CachingTokenizer):
 
         # Check if the tokenizer is cached
         if tokenizer_name not in self._tokenizer_name_to_tokenizer:
+            if self._aleph_alpha_client is None:
+                raise ValueError("Aleph Alpha API key not set.")
             self._tokenizer_name_to_tokenizer[tokenizer_name] = self._aleph_alpha_client.tokenizer(tokenizer_name)
             hlog(f"Initialized tokenizer: {tokenizer_name}")
         return self._tokenizer_name_to_tokenizer[tokenizer_name]

helm/{proxy/tokenizers → tokenizers}/anthropic_tokenizer.py

@@ -15,32 +15,38 @@ except ModuleNotFoundError as e:
 
 class AnthropicTokenizer(CachingTokenizer):
     LOCK: threading.Lock = threading.Lock()
+    """Global lock for the Anthropic tokenizer.
+
+    The Anthropic tokenizer is a wrapper around a single global Hugging Face tokenizer, which is thread-hostile."""
 
     def __init__(self, cache_config: CacheConfig) -> None:
         super().__init__(cache_config)
         with AnthropicTokenizer.LOCK:
             self._tokenizer: PreTrainedTokenizerBase = PreTrainedTokenizerFast(
-                tokenizer_object=anthropic.get_tokenizer()
+                tokenizer_object=anthropic.Anthropic().get_tokenizer()
             )
 
     def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
         if request["encode"]:
             if request["truncation"]:
-                tokens = self._tokenizer.encode(
-                    request["text"],
-                    truncation=request["truncation"],
-                    max_length=request["max_length"],
-                    add_special_tokens=False,
-                )
+                with AnthropicTokenizer.LOCK:
+                    tokens = self._tokenizer.encode(
+                        request["text"],
+                        truncation=request["truncation"],
+                        max_length=request["max_length"],
+                        add_special_tokens=False,
+                    )
             else:
-                tokens = self._tokenizer.encode(request["text"], add_special_tokens=False)
+                with AnthropicTokenizer.LOCK:
+                    tokens = self._tokenizer.encode(request["text"], add_special_tokens=False)
         else:
             # No encoding, just return the token strings
             tokens = [self._tokenizer.convert_tokens_to_string([i]) for i in self._tokenizer.tokenize(request["text"])]
         return {"tokens": tokens}
 
     def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
-        text = self._tokenizer.decode(
-            request["tokens"], clean_up_tokenization_spaces=request["clean_up_tokenization_spaces"]
-        )
+        with AnthropicTokenizer.LOCK:
+            text = self._tokenizer.decode(
+                request["tokens"], clean_up_tokenization_spaces=request["clean_up_tokenization_spaces"]
+            )
         return {"text": text}

helm/tokenizers/auto_tokenizer.py

@@ -0,0 +1,93 @@
+from dataclasses import replace
+from typing import Any, Dict, Mapping, Optional
+
+from retrying import Attempt, RetryError
+
+from helm.benchmark.tokenizer_config_registry import get_tokenizer_config
+from helm.common.credentials_utils import provide_api_key
+from helm.common.cache_backend_config import CacheBackendConfig, CacheConfig
+from helm.common.hierarchical_logger import hlog
+from helm.common.object_spec import create_object, inject_object_spec_args
+from helm.proxy.retry import retry_tokenizer_request
+from helm.common.tokenization_request import (
+    DecodeRequest,
+    DecodeRequestResult,
+    TokenizationRequest,
+    TokenizationRequestResult,
+)
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class AutoTokenizer(Tokenizer):
+    """Automatically dispatch to the proper `Tokenizer` based on the tokenizer name."""
+
+    def __init__(self, credentials: Mapping[str, Any], cache_backend_config: CacheBackendConfig):
+        self.credentials = credentials
+        self.cache_backend_config = cache_backend_config
+        self.tokenizers: Dict[str, Tokenizer] = {}
+        hlog(f"AutoTokenizer: cache_backend_config = {cache_backend_config}")
+
+    def _get_tokenizer(self, tokenizer_name: str) -> Tokenizer:
+        # First try to find the tokenizer in the cache
+        tokenizer: Optional[Tokenizer] = self.tokenizers.get(tokenizer_name)
+        if tokenizer is not None:
+            return tokenizer
+
+        # Otherwise, create the tokenizer
+        organization: str = tokenizer_name.split("/")[0]
+        cache_config: CacheConfig = self.cache_backend_config.get_cache_config(organization)
+
+        tokenizer_config = get_tokenizer_config(tokenizer_name)
+        if tokenizer_config:
+            tokenizer_spec = inject_object_spec_args(
+                tokenizer_config.tokenizer_spec,
+                constant_bindings={"cache_config": cache_config},
+                provider_bindings={
+                    "api_key": lambda: provide_api_key(self.credentials, organization),
+                    "project_id": lambda: self.credentials.get(organization + "ProjectId", None),  # VertexAI
+                    "location": lambda: self.credentials.get(organization + "Location", None),  # VertexAI
+                },
+            )
+            tokenizer = create_object(tokenizer_spec)
+        else:
+            hlog(f"No tokenizer config for {tokenizer_name}")
+
+        # Cache the tokenizer
+        assert isinstance(tokenizer, Tokenizer)  # To make mypy happy
+        self.tokenizers[tokenizer_name] = tokenizer
+
+        return tokenizer
+
+    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
+        """Tokenizes based on the name of the tokenizer (e.g., huggingface/gpt2)."""
+
+        @retry_tokenizer_request
+        def tokenize_with_retry(tokenizer: Tokenizer, request: TokenizationRequest) -> TokenizationRequestResult:
+            return tokenizer.tokenize(request)
+
+        tokenizer: Tokenizer = self._get_tokenizer(request.tokenizer)
+
+        try:
+            return tokenize_with_retry(tokenizer=tokenizer, request=request)
+        except RetryError as e:
+            last_attempt: Attempt = e.last_attempt
+            retry_error: str = f"Failed to tokenize after retrying {last_attempt.attempt_number} times"
+            hlog(retry_error)
+            return replace(last_attempt.value, error=f"{retry_error}. Error: {last_attempt.value.error}")
+
+    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
+        """Decodes based on the the name of the tokenizer (e.g., huggingface/gpt2)."""
+
+        @retry_tokenizer_request
+        def decode_with_retry(tokenizer: Tokenizer, request: DecodeRequest) -> DecodeRequestResult:
+            return tokenizer.decode(request)
+
+        tokenizer: Tokenizer = self._get_tokenizer(request.tokenizer)
+
+        try:
+            return decode_with_retry(tokenizer=tokenizer, request=request)
+        except RetryError as e:
+            last_attempt: Attempt = e.last_attempt
+            retry_error: str = f"Failed to decode after retrying {last_attempt.attempt_number} times"
+            hlog(retry_error)
+            return replace(last_attempt.value, error=f"{retry_error}. Error: {last_attempt.value.error}")

helm/{proxy/tokenizers → tokenizers}/caching_tokenizer.py

@@ -100,9 +100,15 @@ class CachingTokenizer(Tokenizer):
             # Internal check of the type of the first token
             # This is to make sure that the tokenization is correct
             if request.encode and len(tokens) > 0:
-                assert type(tokens[0].value) == int
+                assert type(tokens[0].value) == int, (
+                    f"tokenize() returned strings instead of integers when encode is True: "
+                    f"request={request} repsonse={response}"
+                )
             elif not request.encode and len(tokens) > 0:
-                assert type(tokens[0].value) == str
+                assert type(tokens[0].value) == str, (
+                    f"tokenize() returned integers instead of strings when encode is False: "
+                    f"request={request} repsonse={response}"
+                )
 
             result = TokenizationRequestResult(
                 success=True,

helm/{proxy/tokenizers → tokenizers}/cohere_tokenizer.py

@@ -9,7 +9,7 @@ from helm.common.tokenization_request import (
     DecodeRequestResult,
     TokenizationToken,
 )
-from helm.proxy.clients.cohere_utils import get_cohere_url, DEFAULT_COHERE_API_VERSION
+from helm.clients.cohere_utils import get_cohere_url, DEFAULT_COHERE_API_VERSION
 from .caching_tokenizer import CachingTokenizer
 
 

helm/{proxy/tokenizers → tokenizers}/http_model_tokenizer.py

@@ -1,6 +1,6 @@
 import os
 from dataclasses import asdict
-from typing import Optional
+from typing import Any, Dict, Optional
 
 from helm.common.cache import Cache, CacheConfig
 from helm.common.request import wrap_request_time
@@ -42,7 +42,7 @@ class HTTPModelTokenizer(Tokenizer):
 
         try:
 
-            def do_it():
+            def do_it() -> Dict[str, Any]:
                 url = f"{self.base_url}/tokenize"
                 response = requests.post(url, json=raw_request)
                 response.raise_for_status()
@@ -70,7 +70,7 @@ class HTTPModelTokenizer(Tokenizer):
 
         try:
 
-            def do_it():
+            def do_it() -> Dict[str, Any]:
                 url = f"{self.base_url}/decode"
                 response = requests.post(url, json={"tokens": request.tokens})
                 response.raise_for_status()