crfm-helm 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/proxy/token_counters/auto_token_counter.py

@@ -1,42 +1,42 @@
-from typing import Dict, List
-
-from helm.common.request import Request, Sequence
-from helm.proxy.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
-from .ai21_token_counter import AI21TokenCounter
-from .cohere_token_counter import CohereTokenCounter
-from .free_token_counter import FreeTokenCounter
-from .gooseai_token_counter import GooseAITokenCounter
-from .openai_token_counter import OpenAITokenCounter
+from typing import List
+from helm.benchmark.model_deployment_registry import ModelDeployment, get_model_deployment
+
+from helm.common.request import Request, GeneratedOutput
+from helm.tokenizers.auto_tokenizer import AutoTokenizer
+from helm.common.tokenization_request import TokenizationRequest, TokenizationRequestResult
 from .token_counter import TokenCounter
 
 
 class AutoTokenCounter(TokenCounter):
-    """Automatically count tokens based on the organization."""
-
-    def __init__(self, huggingface_tokenizer: HuggingFaceTokenizer):
-        self.token_counters: Dict[str, TokenCounter] = {}
-        self.huggingface_tokenizer: HuggingFaceTokenizer = huggingface_tokenizer
-
-    def get_token_counter(self, organization: str) -> TokenCounter:
-        """Return a token counter based on the organization."""
-        token_counter = self.token_counters.get(organization)
-        if token_counter is None:
-            if organization == "openai":
-                token_counter = OpenAITokenCounter(self.huggingface_tokenizer)
-            elif organization == "ai21":
-                token_counter = AI21TokenCounter()
-            elif organization == "gooseai":
-                token_counter = GooseAITokenCounter()
-            elif organization == "cohere":
-                token_counter = CohereTokenCounter()
+    """Automatically count tokens based on the model_deployment."""
+
+    def __init__(self, auto_tokenizer: AutoTokenizer):
+        self.auto_tokenizer: AutoTokenizer = auto_tokenizer
+
+    def count_tokens(self, request: Request, completions: List[GeneratedOutput]) -> int:
+        """Counts tokens based on the model deployment.
+
+        This counts the number of tokens in the request and completions.
+        Both input and output tokens are counted. For some model providers,
+        this method will return a larger number of tokens than the actual
+        token count used for billing. For example, GooseAI only charges for
+        (output_tokens - 25) rather than (input_tokens + output_tokens)."""
+        model_deployment: ModelDeployment = get_model_deployment(request.model_deployment)
+        assert model_deployment.tokenizer_name
+        tokenizer_name = model_deployment.tokenizer_name
+
+        num_completion_tokens = 0
+        for completion in completions:
+            if completion.tokens:
+                num_completion_tokens += len(completion.tokens)
             else:
-                token_counter = FreeTokenCounter()
-            self.token_counters[organization] = token_counter
-        return token_counter
-
-    def count_tokens(self, request: Request, completions: List[Sequence]) -> int:
-        """
-        Counts tokens based on the organization.
-        """
-        token_counter: TokenCounter = self.get_token_counter(request.model_host)
-        return token_counter.count_tokens(request, completions)
+                tokenized_completion: TokenizationRequestResult = self.auto_tokenizer.tokenize(
+                    TokenizationRequest(request.prompt, tokenizer=tokenizer_name)
+                )
+                num_completion_tokens += len(tokenized_completion.tokens)
+
+        tokenized_prompt: TokenizationRequestResult = self.auto_tokenizer.tokenize(
+            TokenizationRequest(request.prompt, tokenizer=tokenizer_name)
+        )
+        num_prompt_tokens = len(tokenized_prompt.tokens)
+        return num_prompt_tokens + num_completion_tokens

helm/proxy/token_counters/test_auto_token_counter.py

@@ -0,0 +1,164 @@
+from typing import List
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
+
+from helm.common.request import Request, GeneratedOutput, Token
+from helm.tokenizers.auto_tokenizer import AutoTokenizer
+from helm.proxy.token_counters.auto_token_counter import AutoTokenCounter
+
+
+class TestAutoTokenCounter:
+    def test_count_tokens_openai(self):
+        token_counter = AutoTokenCounter(
+            AutoTokenizer(credentials={}, cache_backend_config=BlackHoleCacheBackendConfig())
+        )
+        # The following prompt has 51 tokens according to the GPT-2 tokenizer
+        request = Request(
+            model="openai/text-davinci-002",
+            model_deployment="openai/text-davinci-002",
+            prompt="The Center for Research on Foundation Models (CRFM) is "
+            "an interdisciplinary initiative born out of the Stanford "
+            "Institute for Human-Centered Artificial Intelligence (HAI) "
+            "that aims to make fundamental advances in the study, development, "
+            "and deployment of foundation models.",
+        )
+        completions: List[GeneratedOutput] = [
+            GeneratedOutput(
+                text=" The CRFM is dedicated to advancing our knowledge of the foundations of artificial intelligence "
+                "(AI) and related fields. It focuses on foundational questions in AI, which are",
+                logprob=-49.00783279519999,
+                tokens=[
+                    Token(text=" The", logprob=-1.8096403),
+                    Token(text=" CR", logprob=-1.2861944),
+                    Token(text="FM", logprob=-0.0032369632),
+                    Token(text=" is", logprob=-1.4355252),
+                    Token(text=" dedicated", logprob=-3.814422),
+                    Token(text=" to", logprob=-0.009623392),
+                    Token(text=" advancing", logprob=-2.6732886),
+                    Token(text=" our", logprob=-3.123714),
+                    Token(text=" knowledge", logprob=-3.030337),
+                    Token(text=" of", logprob=-0.46280858),
+                    Token(text=" the", logprob=-1.4058315),
+                    Token(text=" foundations", logprob=-2.0638132),
+                    Token(text=" of", logprob=-0.2607486),
+                    Token(text=" artificial", logprob=-1.1653417),
+                    Token(text=" intelligence", logprob=-0.03756146),
+                    Token(text=" (", logprob=-2.019812),
+                    Token(text="AI", logprob=-0.03869382),
+                    Token(text=")", logprob=-0.49895737),
+                    Token(text=" and", logprob=-0.81909865),
+                    Token(text=" related", logprob=-2.611718),
+                    Token(text=" fields", logprob=-0.7640527),
+                    Token(text=".", logprob=-1.8066244),
+                    Token(text=" It", logprob=-2.2856107),
+                    Token(text=" focuses", logprob=-3.3735154),
+                    Token(text=" on", logprob=-0.13244776),
+                    Token(text=" foundational", logprob=-1.2640914),
+                    Token(text=" questions", logprob=-2.010647),
+                    Token(text=" in", logprob=-1.980726),
+                    Token(text=" AI", logprob=-0.5709368),
+                    Token(text=",", logprob=-1.036094),
+                    Token(text=" which", logprob=-3.826836),
+                    Token(text=" are", logprob=-1.3858839),
+                ],
+            )
+        ]
+
+        # Verified against https://beta.openai.com/tokenizer. Prompt + completions = 51 + 32.
+        assert token_counter.count_tokens(request, completions) == 51 + 32
+
+    def test_count_tokens_anthropic(self):
+        token_counter = AutoTokenCounter(
+            AutoTokenizer(credentials={}, cache_backend_config=BlackHoleCacheBackendConfig())
+        )
+        request = Request(
+            model="anthropic/claude-instant-v1",
+            model_deployment="anthropic/claude-instant-v1",
+            prompt="\n\nHuman:The Center for Research on Foundation Models (CRFM) is "
+            "an interdisciplinary initiative born out of the Stanford "
+            "Institute for Human-Centered Artificial Intelligence (HAI) "
+            "that aims to make fundamental advances in the study, development, "
+            "and deployment of foundation models.\n\nAssistant:",
+        )
+        completions: List[GeneratedOutput] = [
+            GeneratedOutput(
+                text="Thank you for the background information. The Center for Research "
+                "on Foundation Models sounds like an interesting initiative focused on "
+                "advancing research and responsible development of large AI models. I "
+                "don't have any personal thoughts on it, but I'm happy to discuss or "
+                "provide more information if helpful. As an AI assistant, I don't have "
+                "subjective opinions.",
+                logprob=0,
+                tokens=[
+                    Token(text="Thank", logprob=0),
+                    Token(text=" you", logprob=0),
+                    Token(text=" for", logprob=0),
+                    Token(text=" the", logprob=0),
+                    Token(text=" background", logprob=0),
+                    Token(text=" information", logprob=0),
+                    Token(text=".", logprob=0),
+                    Token(text=" The", logprob=0),
+                    Token(text=" Center", logprob=0),
+                    Token(text=" for", logprob=0),
+                    Token(text=" Research", logprob=0),
+                    Token(text=" on", logprob=0),
+                    Token(text=" Foundation", logprob=0),
+                    Token(text=" Models", logprob=0),
+                    Token(text=" sounds", logprob=0),
+                    Token(text=" like", logprob=0),
+                    Token(text=" an", logprob=0),
+                    Token(text=" interesting", logprob=0),
+                    Token(text=" initiative", logprob=0),
+                    Token(text=" focused", logprob=0),
+                    Token(text=" on", logprob=0),
+                    Token(text=" advancing", logprob=0),
+                    Token(text=" research", logprob=0),
+                    Token(text=" and", logprob=0),
+                    Token(text=" responsible", logprob=0),
+                    Token(text=" development", logprob=0),
+                    Token(text=" of", logprob=0),
+                    Token(text=" large", logprob=0),
+                    Token(text=" AI", logprob=0),
+                    Token(text=" models", logprob=0),
+                    Token(text=".", logprob=0),
+                    Token(text=" I", logprob=0),
+                    Token(text=" don", logprob=0),
+                    Token(text="'t", logprob=0),
+                    Token(text=" have", logprob=0),
+                    Token(text=" any", logprob=0),
+                    Token(text=" personal", logprob=0),
+                    Token(text=" thoughts", logprob=0),
+                    Token(text=" on", logprob=0),
+                    Token(text=" it", logprob=0),
+                    Token(text=",", logprob=0),
+                    Token(text=" but", logprob=0),
+                    Token(text=" I", logprob=0),
+                    Token(text="'m", logprob=0),
+                    Token(text=" happy", logprob=0),
+                    Token(text=" to", logprob=0),
+                    Token(text=" discuss", logprob=0),
+                    Token(text=" or", logprob=0),
+                    Token(text=" provide", logprob=0),
+                    Token(text=" more", logprob=0),
+                    Token(text=" information", logprob=0),
+                    Token(text=" if", logprob=0),
+                    Token(text=" helpful", logprob=0),
+                    Token(text=".", logprob=0),
+                    Token(text=" As", logprob=0),
+                    Token(text=" an", logprob=0),
+                    Token(text=" AI", logprob=0),
+                    Token(text=" assistant", logprob=0),
+                    Token(text=",", logprob=0),
+                    Token(text=" I", logprob=0),
+                    Token(text=" don", logprob=0),
+                    Token(text="'t", logprob=0),
+                    Token(text=" have", logprob=0),
+                    Token(text=" subjective", logprob=0),
+                    Token(text=" opinions", logprob=0),
+                    Token(text=".", logprob=0),
+                ],
+                finish_reason=None,
+                multimodal_content=None,
+            )
+        ]
+
+        assert token_counter.count_tokens(request, completions) == 126

helm/proxy/token_counters/token_counter.py

@@ -1,15 +1,13 @@
 from abc import ABC, abstractmethod
 from typing import List
 
-from helm.common.request import Sequence, Request
+from helm.common.request import GeneratedOutput, Request
 
 
 class TokenCounter(ABC):
     """Counts the number of tokens used given `Request` and completions."""
 
     @abstractmethod
-    def count_tokens(self, request: Request, completions: List[Sequence]) -> int:
-        """
-        Counts the total number of tokens given a request and completions.
-        """
+    def count_tokens(self, request: Request, completions: List[GeneratedOutput]) -> int:
+        """Counts the total number of tokens given a request and completions."""
         pass

helm/{proxy/tokenizers → tokenizers}/ai21_tokenizer.py

@@ -1,4 +1,4 @@
-from typing import Dict, List
+from typing import Any, Dict, List
 import requests
 
 from dacite import from_dict
@@ -12,7 +12,7 @@ from helm.common.tokenization_request import (
     DecodeRequest,
     DecodeRequestResult,
 )
-from helm.proxy.clients.ai21_utils import AI21RequestError, handle_failed_request
+from helm.clients.ai21_utils import AI21RequestError, handle_failed_request
 from .tokenizer import Tokenizer
 
 
@@ -28,7 +28,7 @@ class AI21Tokenizer(Tokenizer):
         # TODO: Does not support encoding
         raw_request: Dict[str, str] = {"text": request.text}
 
-        def do_it():
+        def do_it() -> Dict[str, Any]:
             response = requests.post(
                 "https://api.ai21.com/studio/v1/tokenize",
                 headers={"Authorization": f"Bearer {self.api_key}"},

helm/{proxy/tokenizers → tokenizers}/anthropic_tokenizer.py

@@ -23,7 +23,7 @@ class AnthropicTokenizer(CachingTokenizer):
         super().__init__(cache_config)
         with AnthropicTokenizer.LOCK:
             self._tokenizer: PreTrainedTokenizerBase = PreTrainedTokenizerFast(
-                tokenizer_object=anthropic.get_tokenizer()
+                tokenizer_object=anthropic.Anthropic().get_tokenizer()
             )
 
     def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:

helm/{proxy/tokenizers → tokenizers}/auto_tokenizer.py

@@ -4,9 +4,8 @@ from typing import Any, Dict, Mapping, Optional
 from retrying import Attempt, RetryError
 
 from helm.benchmark.tokenizer_config_registry import get_tokenizer_config
-from helm.common.cache_utils import build_cache_config
 from helm.common.credentials_utils import provide_api_key
-from helm.common.cache import CacheConfig
+from helm.common.cache_backend_config import CacheBackendConfig, CacheConfig
 from helm.common.hierarchical_logger import hlog
 from helm.common.object_spec import create_object, inject_object_spec_args
 from helm.proxy.retry import retry_tokenizer_request
@@ -16,19 +15,17 @@ from helm.common.tokenization_request import (
     TokenizationRequest,
     TokenizationRequestResult,
 )
-from helm.proxy.tokenizers.tokenizer import Tokenizer
+from helm.tokenizers.tokenizer import Tokenizer
 
 
 class AutoTokenizer(Tokenizer):
     """Automatically dispatch to the proper `Tokenizer` based on the tokenizer name."""
 
-    def __init__(self, credentials: Mapping[str, Any], cache_path: str, mongo_uri: str = ""):
+    def __init__(self, credentials: Mapping[str, Any], cache_backend_config: CacheBackendConfig):
         self.credentials = credentials
-        self.cache_path = cache_path
-        self.mongo_uri = mongo_uri
+        self.cache_backend_config = cache_backend_config
         self.tokenizers: Dict[str, Tokenizer] = {}
-        hlog(f"AutoTokenizer: cache_path = {cache_path}")
-        hlog(f"AutoTokenizer: mongo_uri = {mongo_uri}")
+        hlog(f"AutoTokenizer: cache_backend_config = {cache_backend_config}")
 
     def _get_tokenizer(self, tokenizer_name: str) -> Tokenizer:
         # First try to find the tokenizer in the cache
@@ -38,7 +35,7 @@ class AutoTokenizer(Tokenizer):
 
         # Otherwise, create the tokenizer
         organization: str = tokenizer_name.split("/")[0]
-        cache_config: CacheConfig = build_cache_config(self.cache_path, self.mongo_uri, organization)
+        cache_config: CacheConfig = self.cache_backend_config.get_cache_config(organization)
 
         tokenizer_config = get_tokenizer_config(tokenizer_name)
         if tokenizer_config:

helm/{proxy/tokenizers → tokenizers}/cohere_tokenizer.py

@@ -9,7 +9,7 @@ from helm.common.tokenization_request import (
     DecodeRequestResult,
     TokenizationToken,
 )
-from helm.proxy.clients.cohere_utils import get_cohere_url, DEFAULT_COHERE_API_VERSION
+from helm.clients.cohere_utils import get_cohere_url, DEFAULT_COHERE_API_VERSION
 from .caching_tokenizer import CachingTokenizer
 
 

helm/{proxy/tokenizers → tokenizers}/http_model_tokenizer.py

@@ -1,6 +1,6 @@
 import os
 from dataclasses import asdict
-from typing import Optional
+from typing import Any, Dict, Optional
 
 from helm.common.cache import Cache, CacheConfig
 from helm.common.request import wrap_request_time
@@ -42,7 +42,7 @@ class HTTPModelTokenizer(Tokenizer):
 
         try:
 
-            def do_it():
+            def do_it() -> Dict[str, Any]:
                 url = f"{self.base_url}/tokenize"
                 response = requests.post(url, json=raw_request)
                 response.raise_for_status()
@@ -70,7 +70,7 @@ class HTTPModelTokenizer(Tokenizer):
 
         try:
 
-            def do_it():
+            def do_it() -> Dict[str, Any]:
                 url = f"{self.base_url}/decode"
                 response = requests.post(url, json={"tokens": request.tokens})
                 response.raise_for_status()

helm/{proxy/tokenizers → tokenizers}/huggingface_tokenizer.py

@@ -1,5 +1,5 @@
 import os
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, cast
 from threading import Lock
 from helm.common.cache import CacheConfig
 from helm.common.concurrency import ThreadSafeWrapper
@@ -11,26 +11,6 @@ from .caching_tokenizer import CachingTokenizer
 from .tokenizer import cleanup_tokens
 
 
-# TODO: Delete this.
-_MODEL_NAME_ALIASES: Dict[str, str] = {
-    "google/t5-11b": "t5-11b",
-    "huggingface/gpt2": "gpt2",
-    "huggingface/santacoder": "bigcode/santacoder",
-    "huggingface/starcoder": "bigcode/starcoder",
-    "writer/gpt2": "gpt2",  # Palmyra models do not support echo
-    # So they have a different TokenizerConfig called "writer/gpt2"
-    # when in reality they use the same tokenizer as "huggingface/gpt2"
-    "microsoft/gpt2": "gpt2",  # Same as above
-}
-"""Mapping of some HELM model names to Hugging Face pretrained model name."""
-
-
-# TODO: Delete this.
-def resolve_alias(model_name: str) -> str:
-    """Resolve some HELM model names to Hugging Face pretrained model name."""
-    return _MODEL_NAME_ALIASES.get(model_name, model_name)
-
-
 WrappedPreTrainedTokenizer = ThreadSafeWrapper[PreTrainedTokenizerBase]
 """Thread safe wrapper around Hugging Face PreTrainedTokenizerBase.
 
@@ -106,11 +86,9 @@ class HuggingFaceTokenizer(CachingTokenizer):
 
     def _get_tokenizer_for_request(self, request: Dict[str, Any]) -> WrappedPreTrainedTokenizer:
         """Method used in both _tokenize_do_it and _decode_do_it to get the tokenizer."""
-        pretrained_model_name_or_path: str
-        if self._pretrained_model_name_or_path:
-            pretrained_model_name_or_path = self._pretrained_model_name_or_path
-        else:
-            pretrained_model_name_or_path = resolve_alias(request["tokenizer"])
+        pretrained_model_name_or_path = (
+            self._pretrained_model_name_or_path if self._pretrained_model_name_or_path else request["tokenizer"]
+        )
         return HuggingFaceTokenizer.get_tokenizer(
             helm_tokenizer_name=request["tokenizer"],
             pretrained_model_name_or_path=pretrained_model_name_or_path,
@@ -155,6 +133,9 @@ class HuggingFaceTokenizer(CachingTokenizer):
                 # Just like tokenize("Hello", encode=False) would return ["Hello"].
                 with self._get_tokenizer_for_request(request) as tokenizer:
                     tokens = tokenizer.tokenize(request["text"])
+                # Some tokenizers (e.g. Qwen/Qwen-7B) return the tokens as bytes, so we have to decode them to strings.
+                if tokens and type(tokens[0]) == bytes:
+                    tokens = [cast(bytes, token).decode(errors="ignore") for token in tokens]
                 tokens = cleanup_tokens(tokens, request["tokenizer"])
         return {"tokens": tokens}
 

helm/tokenizers/simple_tokenizer.py

@@ -0,0 +1,33 @@
+from helm.common.tokenization_request import (
+    DecodeRequest,
+    DecodeRequestResult,
+    TokenizationRequest,
+    TokenizationRequestResult,
+    TokenizationToken,
+)
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class SimpleTokenizer(Tokenizer):
+    """Simple tokenizer for tutorials and for debugging."""
+
+    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
+        if request.encode:
+            return TokenizationRequestResult(
+                success=True,
+                cached=False,
+                tokens=[TokenizationToken(ord(character)) for character in request.text],
+                text=request.text,
+            )
+        else:
+            return TokenizationRequestResult(
+                success=True,
+                cached=False,
+                tokens=[TokenizationToken(character) for character in request.text],
+                text=request.text,
+            )
+
+    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
+        return DecodeRequestResult(
+            success=True, cached=False, text="".join([chr(code_point) for code_point in request.tokens])
+        )

helm/{proxy/tokenizers → tokenizers}/test_anthropic_tokenizer.py

@@ -10,7 +10,7 @@ from helm.common.tokenization_request import (
     TokenizationRequest,
     TokenizationRequestResult,
 )
-from helm.proxy.tokenizers.anthropic_tokenizer import AnthropicTokenizer
+from helm.tokenizers.anthropic_tokenizer import AnthropicTokenizer
 
 
 class TestAnthropicTokenizer:

helm/{proxy/tokenizers → tokenizers}/test_huggingface_tokenizer.py

@@ -124,6 +124,9 @@ class TestHuggingFaceTokenizer:
     def test_get_santacoder(self):
         TestHuggingFaceTokenizer.verify_get_tokenizer("bigcode/santacoder", 62)
 
+    def test_get_clip_tokenizer(self):
+        TestHuggingFaceTokenizer.verify_get_tokenizer("openai/clip-vit-large-patch14", 50)
+
     def test_gpt2_tokenize_eos(self):
         eos_token: str = "<|endoftext|>"
         wrapped_tokenizer = HuggingFaceTokenizer.get_tokenizer("huggingface/gpt2", pretrained_model_name_or_path="gpt2")

helm/tokenizers/test_simple_tokenizer.py

@@ -0,0 +1,33 @@
+from helm.common.tokenization_request import (
+    DecodeRequest,
+    TokenizationRequest,
+    TokenizationToken,
+)
+from helm.tokenizers.simple_tokenizer import SimpleTokenizer
+
+
+def test_simple_tokenizer_tokenize():
+    tokenizer = SimpleTokenizer()
+    request = TokenizationRequest(tokenizer="simple/tokenizer1", text="otter 🦦")
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in ["o", "t", "t", "e", "r", " ", "🦦"]]
+
+
+def test_simple_tokenizer_encode():
+    tokenizer = SimpleTokenizer()
+    request = TokenizationRequest(tokenizer="simple/tokenizer1", text="otter 🦦", encode=True)
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in [111, 116, 116, 101, 114, 32, 129446]]
+
+
+def test_simple_tokenizer_decode():
+    tokenizer = SimpleTokenizer()
+    request = DecodeRequest(tokenizer="simple/tokenizer1", tokens=[111, 116, 116, 101, 114, 32, 129446])
+    result = tokenizer.decode(request)
+    assert result.success
+    assert not result.cached
+    assert result.text == "otter 🦦"

helm/{proxy/tokenizers → tokenizers}/vertexai_tokenizer.py

@@ -9,7 +9,7 @@ from helm.common.tokenization_request import (
     TokenizationRequest,
     TokenizationToken,
 )
-from helm.proxy.tokenizers.caching_tokenizer import CachingTokenizer
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
 from helm.proxy.retry import NonRetriableException
 
 try:

helm/{proxy/tokenizers → tokenizers}/yalm_tokenizer.py

@@ -21,9 +21,11 @@ class YaLMTokenizer(CachingTokenizer):
         # This is a problem because then tokenize(" Hello", encode=False) == tokenize("Hello", encode=False)
         # That is why we manually replace "▁" with a space.
         return {
-            "tokens": token_ids
-            if request["encode"]
-            else cleanup_tokens(self._tokenizer.convert_ids_to_tokens(token_ids), request["tokenizer"])
+            "tokens": (
+                token_ids
+                if request["encode"]
+                else cleanup_tokens(self._tokenizer.convert_ids_to_tokens(token_ids), request["tokenizer"])
+            )
         }
 
     def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:

helm/{proxy/tokenizers → tokenizers}/yalm_tokenizer_data/yalm_tokenizer.py

@@ -16,7 +16,7 @@ adapted from https://github.com/yandex/YaLM-100B/blob/main/megatron_lm/megatron/
 """
 
 
-YALM_TOKENIZER_PACKAGE: str = "helm.proxy.tokenizers.yalm_tokenizer_data"
+YALM_TOKENIZER_PACKAGE: str = "helm.tokenizers.yalm_tokenizer_data"
 YALM_TOKENIZER_VOCAB_FILENAME: str = "voc_100b.sp"