crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/proxy/token_counters/test_auto_token_counter.py

@@ -0,0 +1,164 @@
+from typing import List
+from helm.common.cache_backend_config import BlackHoleCacheBackendConfig
+
+from helm.common.request import Request, GeneratedOutput, Token
+from helm.tokenizers.auto_tokenizer import AutoTokenizer
+from helm.proxy.token_counters.auto_token_counter import AutoTokenCounter
+
+
+class TestAutoTokenCounter:
+    def test_count_tokens_openai(self):
+        token_counter = AutoTokenCounter(
+            AutoTokenizer(credentials={}, cache_backend_config=BlackHoleCacheBackendConfig())
+        )
+        # The following prompt has 51 tokens according to the GPT-2 tokenizer
+        request = Request(
+            model="openai/gpt2",
+            model_deployment="huggingface/gpt2",
+            prompt="The Center for Research on Foundation Models (CRFM) is "
+            "an interdisciplinary initiative born out of the Stanford "
+            "Institute for Human-Centered Artificial Intelligence (HAI) "
+            "that aims to make fundamental advances in the study, development, "
+            "and deployment of foundation models.",
+        )
+        completions: List[GeneratedOutput] = [
+            GeneratedOutput(
+                text=" The CRFM is dedicated to advancing our knowledge of the foundations of artificial intelligence "
+                "(AI) and related fields. It focuses on foundational questions in AI, which are",
+                logprob=-49.00783279519999,
+                tokens=[
+                    Token(text=" The", logprob=-1.8096403),
+                    Token(text=" CR", logprob=-1.2861944),
+                    Token(text="FM", logprob=-0.0032369632),
+                    Token(text=" is", logprob=-1.4355252),
+                    Token(text=" dedicated", logprob=-3.814422),
+                    Token(text=" to", logprob=-0.009623392),
+                    Token(text=" advancing", logprob=-2.6732886),
+                    Token(text=" our", logprob=-3.123714),
+                    Token(text=" knowledge", logprob=-3.030337),
+                    Token(text=" of", logprob=-0.46280858),
+                    Token(text=" the", logprob=-1.4058315),
+                    Token(text=" foundations", logprob=-2.0638132),
+                    Token(text=" of", logprob=-0.2607486),
+                    Token(text=" artificial", logprob=-1.1653417),
+                    Token(text=" intelligence", logprob=-0.03756146),
+                    Token(text=" (", logprob=-2.019812),
+                    Token(text="AI", logprob=-0.03869382),
+                    Token(text=")", logprob=-0.49895737),
+                    Token(text=" and", logprob=-0.81909865),
+                    Token(text=" related", logprob=-2.611718),
+                    Token(text=" fields", logprob=-0.7640527),
+                    Token(text=".", logprob=-1.8066244),
+                    Token(text=" It", logprob=-2.2856107),
+                    Token(text=" focuses", logprob=-3.3735154),
+                    Token(text=" on", logprob=-0.13244776),
+                    Token(text=" foundational", logprob=-1.2640914),
+                    Token(text=" questions", logprob=-2.010647),
+                    Token(text=" in", logprob=-1.980726),
+                    Token(text=" AI", logprob=-0.5709368),
+                    Token(text=",", logprob=-1.036094),
+                    Token(text=" which", logprob=-3.826836),
+                    Token(text=" are", logprob=-1.3858839),
+                ],
+            )
+        ]
+
+        # Verified against https://beta.openai.com/tokenizer. Prompt + completions = 51 + 32.
+        assert token_counter.count_tokens(request, completions) == 51 + 32
+
+    def test_count_tokens_anthropic(self):
+        token_counter = AutoTokenCounter(
+            AutoTokenizer(credentials={}, cache_backend_config=BlackHoleCacheBackendConfig())
+        )
+        request = Request(
+            model="anthropic/claude-instant-v1",
+            model_deployment="anthropic/claude-instant-v1",
+            prompt="\n\nHuman:The Center for Research on Foundation Models (CRFM) is "
+            "an interdisciplinary initiative born out of the Stanford "
+            "Institute for Human-Centered Artificial Intelligence (HAI) "
+            "that aims to make fundamental advances in the study, development, "
+            "and deployment of foundation models.\n\nAssistant:",
+        )
+        completions: List[GeneratedOutput] = [
+            GeneratedOutput(
+                text="Thank you for the background information. The Center for Research "
+                "on Foundation Models sounds like an interesting initiative focused on "
+                "advancing research and responsible development of large AI models. I "
+                "don't have any personal thoughts on it, but I'm happy to discuss or "
+                "provide more information if helpful. As an AI assistant, I don't have "
+                "subjective opinions.",
+                logprob=0,
+                tokens=[
+                    Token(text="Thank", logprob=0),
+                    Token(text=" you", logprob=0),
+                    Token(text=" for", logprob=0),
+                    Token(text=" the", logprob=0),
+                    Token(text=" background", logprob=0),
+                    Token(text=" information", logprob=0),
+                    Token(text=".", logprob=0),
+                    Token(text=" The", logprob=0),
+                    Token(text=" Center", logprob=0),
+                    Token(text=" for", logprob=0),
+                    Token(text=" Research", logprob=0),
+                    Token(text=" on", logprob=0),
+                    Token(text=" Foundation", logprob=0),
+                    Token(text=" Models", logprob=0),
+                    Token(text=" sounds", logprob=0),
+                    Token(text=" like", logprob=0),
+                    Token(text=" an", logprob=0),
+                    Token(text=" interesting", logprob=0),
+                    Token(text=" initiative", logprob=0),
+                    Token(text=" focused", logprob=0),
+                    Token(text=" on", logprob=0),
+                    Token(text=" advancing", logprob=0),
+                    Token(text=" research", logprob=0),
+                    Token(text=" and", logprob=0),
+                    Token(text=" responsible", logprob=0),
+                    Token(text=" development", logprob=0),
+                    Token(text=" of", logprob=0),
+                    Token(text=" large", logprob=0),
+                    Token(text=" AI", logprob=0),
+                    Token(text=" models", logprob=0),
+                    Token(text=".", logprob=0),
+                    Token(text=" I", logprob=0),
+                    Token(text=" don", logprob=0),
+                    Token(text="'t", logprob=0),
+                    Token(text=" have", logprob=0),
+                    Token(text=" any", logprob=0),
+                    Token(text=" personal", logprob=0),
+                    Token(text=" thoughts", logprob=0),
+                    Token(text=" on", logprob=0),
+                    Token(text=" it", logprob=0),
+                    Token(text=",", logprob=0),
+                    Token(text=" but", logprob=0),
+                    Token(text=" I", logprob=0),
+                    Token(text="'m", logprob=0),
+                    Token(text=" happy", logprob=0),
+                    Token(text=" to", logprob=0),
+                    Token(text=" discuss", logprob=0),
+                    Token(text=" or", logprob=0),
+                    Token(text=" provide", logprob=0),
+                    Token(text=" more", logprob=0),
+                    Token(text=" information", logprob=0),
+                    Token(text=" if", logprob=0),
+                    Token(text=" helpful", logprob=0),
+                    Token(text=".", logprob=0),
+                    Token(text=" As", logprob=0),
+                    Token(text=" an", logprob=0),
+                    Token(text=" AI", logprob=0),
+                    Token(text=" assistant", logprob=0),
+                    Token(text=",", logprob=0),
+                    Token(text=" I", logprob=0),
+                    Token(text=" don", logprob=0),
+                    Token(text="'t", logprob=0),
+                    Token(text=" have", logprob=0),
+                    Token(text=" subjective", logprob=0),
+                    Token(text=" opinions", logprob=0),
+                    Token(text=".", logprob=0),
+                ],
+                finish_reason=None,
+                multimodal_content=None,
+            )
+        ]
+
+        assert token_counter.count_tokens(request, completions) == 126

helm/proxy/token_counters/token_counter.py

@@ -1,15 +1,13 @@
 from abc import ABC, abstractmethod
 from typing import List
 
-from helm.common.request import Sequence, Request
+from helm.common.request import GeneratedOutput, Request
 
 
 class TokenCounter(ABC):
     """Counts the number of tokens used given `Request` and completions."""
 
     @abstractmethod
-    def count_tokens(self, request: Request, completions: List[Sequence]) -> int:
-        """
-        Counts the total number of tokens given a request and completions.
-        """
+    def count_tokens(self, request: Request, completions: List[GeneratedOutput]) -> int:
+        """Counts the total number of tokens given a request and completions."""
         pass

helm/tokenizers/ai21_tokenizer.py

@@ -0,0 +1,52 @@
+import threading
+from typing import Any, Dict
+
+from helm.common.cache import CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
+
+try:
+    from ai21_tokenizer import Tokenizer as SDKTokenizer
+    from ai21_tokenizer.base_tokenizer import BaseTokenizer
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["ai21"])
+
+
+class AI21LocalTokenizer(CachingTokenizer):
+    """AI21 tokenizer using the AI21 Python library."""
+
+    def __init__(self, cache_config: CacheConfig) -> None:
+        super().__init__(cache_config)
+        self._tokenizers_lock = threading.Lock()
+        self.tokenizers: Dict[str, BaseTokenizer] = {}
+
+    def _get_tokenizer(self, tokenizer_name: str) -> BaseTokenizer:
+        with self._tokenizers_lock:
+            if tokenizer_name not in self.tokenizers:
+                self.tokenizers[tokenizer_name] = SDKTokenizer.get_tokenizer(tokenizer_name)
+            return self.tokenizers[tokenizer_name]
+
+    def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        tokenizer_name = request["tokenizer"].split("/")[1]
+        tokenizer = self._get_tokenizer(tokenizer_name)
+        if request["truncation"]:
+            token_ids = tokenizer.encode(
+                text=request["text"],
+                truncation=request["truncation"],
+                max_length=request["max_length"],
+                add_special_tokens=False,
+            )
+        else:
+            token_ids = tokenizer.encode(
+                text=request["text"],
+                add_special_tokens=False,
+            )
+        if request["encode"]:
+            return {"tokens": token_ids}
+        else:
+            return {"tokens": tokenizer.convert_ids_to_tokens(token_ids)}
+
+    def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        tokenizer_name = request["tokenizer"].split("/")[1]
+        tokenizer = self._get_tokenizer(tokenizer_name)
+        return {"text": tokenizer.decode(request["tokens"])}

helm/{proxy/tokenizers → tokenizers}/aleph_alpha_tokenizer.py

@@ -10,7 +10,7 @@ from helm.common.tokenization_request import (
     DecodeRequest,
     TokenizationToken,
 )
-from .caching_tokenizer import CachingTokenizer
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
 
 try:
     from aleph_alpha_client import Client as AlephAlphaPythonClient

helm/{proxy/tokenizers → tokenizers}/auto_tokenizer.py

@@ -4,31 +4,28 @@ from typing import Any, Dict, Mapping, Optional
 from retrying import Attempt, RetryError
 
 from helm.benchmark.tokenizer_config_registry import get_tokenizer_config
-from helm.common.cache_utils import build_cache_config
 from helm.common.credentials_utils import provide_api_key
-from helm.common.cache import CacheConfig
+from helm.common.cache_backend_config import CacheBackendConfig, CacheConfig
 from helm.common.hierarchical_logger import hlog
 from helm.common.object_spec import create_object, inject_object_spec_args
-from helm.proxy.retry import retry_tokenizer_request
+from helm.proxy.retry import NonRetriableException, retry_tokenizer_request
 from helm.common.tokenization_request import (
     DecodeRequest,
     DecodeRequestResult,
     TokenizationRequest,
     TokenizationRequestResult,
 )
-from helm.proxy.tokenizers.tokenizer import Tokenizer
+from helm.tokenizers.tokenizer import Tokenizer
 
 
 class AutoTokenizer(Tokenizer):
     """Automatically dispatch to the proper `Tokenizer` based on the tokenizer name."""
 
-    def __init__(self, credentials: Mapping[str, Any], cache_path: str, mongo_uri: str = ""):
+    def __init__(self, credentials: Mapping[str, Any], cache_backend_config: CacheBackendConfig):
         self.credentials = credentials
-        self.cache_path = cache_path
-        self.mongo_uri = mongo_uri
+        self.cache_backend_config = cache_backend_config
         self.tokenizers: Dict[str, Tokenizer] = {}
-        hlog(f"AutoTokenizer: cache_path = {cache_path}")
-        hlog(f"AutoTokenizer: mongo_uri = {mongo_uri}")
+        hlog(f"AutoTokenizer: cache_backend_config = {cache_backend_config}")
 
     def _get_tokenizer(self, tokenizer_name: str) -> Tokenizer:
         # First try to find the tokenizer in the cache
@@ -38,13 +35,13 @@ class AutoTokenizer(Tokenizer):
 
         # Otherwise, create the tokenizer
         organization: str = tokenizer_name.split("/")[0]
-        cache_config: CacheConfig = build_cache_config(self.cache_path, self.mongo_uri, organization)
+        cache_config: CacheConfig = self.cache_backend_config.get_cache_config(organization)
 
         tokenizer_config = get_tokenizer_config(tokenizer_name)
         if tokenizer_config:
             tokenizer_spec = inject_object_spec_args(
                 tokenizer_config.tokenizer_spec,
-                constant_bindings={"cache_config": cache_config},
+                constant_bindings={"cache_config": cache_config, "tokenizer_name": tokenizer_name},
                 provider_bindings={
                     "api_key": lambda: provide_api_key(self.credentials, organization),
                     "project_id": lambda: self.credentials.get(organization + "ProjectId", None),  # VertexAI
@@ -53,7 +50,7 @@ class AutoTokenizer(Tokenizer):
             )
             tokenizer = create_object(tokenizer_spec)
         else:
-            hlog(f"No tokenizer config for {tokenizer_name}")
+            raise NonRetriableException(f"Could not find tokenizer config for {tokenizer_name}")
 
         # Cache the tokenizer
         assert isinstance(tokenizer, Tokenizer)  # To make mypy happy

helm/{proxy/tokenizers → tokenizers}/caching_tokenizer.py

@@ -1,6 +1,6 @@
 from abc import abstractmethod
 from dataclasses import asdict
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List
 
 from helm.common.cache import Cache, CacheConfig
 from helm.common.request import wrap_request_time
@@ -11,7 +11,7 @@ from helm.common.tokenization_request import (
     DecodeRequestResult,
     TokenizationToken,
 )
-from .tokenizer import Tokenizer
+from helm.tokenizers.tokenizer import Tokenizer
 
 
 class CachingTokenizer(Tokenizer):
@@ -153,31 +153,3 @@ class CachingTokenizer(Tokenizer):
             )
         except Exception as error:
             raise ValueError(f"Failed to decode tokens with {self.__class__.__name__} tokenizer: {error}") from error
-
-
-def cleanup_str(token: str, tokenizer_name: Optional[str] = None) -> str:
-    """
-    Certain tokenizers introduce special characters to represent spaces, such as
-    "Ġ" or "▁". This function removes those characters.
-    """
-    if tokenizer_name in [
-        "TsinghuaKEG/ice",
-        "bigscience/T0pp",
-        "google/t5-11b",
-        "google/flan-t5-xxl",
-        "google/ul2",
-        "Yandex/yalm",
-        "ai21/j1",
-        "together",
-    ]:
-        return token.replace("▁", " ")
-    elif tokenizer_name is not None and tokenizer_name.startswith("huggingface"):
-        return token.replace("Ġ", " ")
-    return token
-
-
-def cleanup_tokens(tokens: List[str], tokenizer_name: Optional[str] = None) -> List[str]:
-    """
-    Applies `cleanup_str` to each token in `tokens`.
-    """
-    return [cleanup_str(token, tokenizer_name) for token in tokens]

helm/tokenizers/cohere_tokenizer.py

@@ -0,0 +1,50 @@
+from typing import Any, Dict, List, Optional
+
+import cohere
+from cohere.manually_maintained.tokenizers import get_hf_tokenizer
+
+from helm.common.cache import CacheConfig
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationToken,
+)
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
+
+
+class CohereLocalTokenizer(CachingTokenizer):
+    """Cohere tokenizer using the Cohere Python library."""
+
+    def __init__(self, api_key: Optional[str], cache_config: CacheConfig) -> None:
+        super().__init__(cache_config)
+        self.client = cohere.Client(api_key)
+
+    def _tokenization_request_to_cache_key(self, request: TokenizationRequest) -> Dict[str, Any]:
+        return {"text": request.text, "tokenizer": request.tokenizer}
+
+    def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        model: str = request["tokenizer"].split("/")[1]
+        # Workaround for https://github.com/cohere-ai/cohere-python/issues/493
+        # `token_strings` are always set to `[]`, so we have to populate it ourselves.
+        response = self.client.tokenize(text=request["text"], model=model)
+        response_dict = response.dict()
+        response_dict["token_strings"] = get_hf_tokenizer(self.client, model).decode_batch(
+            [[token] for token in response.tokens]
+        )
+        return response_dict
+
+    def _tokenization_raw_response_to_tokens(
+        self, response: Dict[str, Any], request: TokenizationRequest
+    ) -> List[TokenizationToken]:
+        tokens: List[TokenizationToken] = []
+        if request.encode:
+            tokens = [TokenizationToken(token) for token in response["tokens"]]
+        else:
+            tokens = [TokenizationToken(token) for token in response["token_strings"]]
+        if request.truncation:
+            tokens = tokens[: request.max_length]
+        return tokens
+
+    def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        model: str = request["tokenizer"].split("/")[1]
+        response = self.client.detokenize(tokens=request["tokens"], model=model)
+        return response.dict()

helm/tokenizers/grok_tokenizer.py

@@ -0,0 +1,55 @@
+import dataclasses
+import os
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from helm.common.cache import CacheConfig
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationToken,
+)
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
+
+
+class GrokAPITokenizer(CachingTokenizer):
+    """Tokenizer that uses the xAI Grok Tokenize Text API
+
+    Doc: https://docs.x.ai/docs/api-reference#tokenize-text"""
+
+    def __init__(self, cache_config: CacheConfig, api_key: Optional[str] = None) -> None:
+        super().__init__(cache_config)
+        self.api_key = api_key or os.environ.get("XAI_API_KEY")
+
+    def _tokenization_request_to_cache_key(self, request: TokenizationRequest) -> Dict[str, Any]:
+        cache_key = dataclasses.asdict(request)
+        # Delete encode because the Grok API simulateously gives string and integer tokens.
+        del cache_key["encode"]
+        return cache_key
+
+    def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.api_key:
+            raise Exception(
+                "No Grok API key found. "
+                "Set grokApiKey in credentials.conf or set the GROK_API_KEY environment variable"
+            )
+        text = request["text"]
+        if not text:
+            return {"token_ids": []}
+        model = request["tokenizer"].split("/")[-1]
+        response = requests.post(
+            url="https://api.x.ai/v1/tokenize-text",
+            headers={"Authorization": f"Bearer {self.api_key}"},
+            json={"text": text, "model": model},
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def _tokenization_raw_response_to_tokens(
+        self, response: Dict[str, Any], request: TokenizationRequest
+    ) -> List[TokenizationToken]:
+        raw_token_field_name = "token_id" if request.encode else "string_token"
+        return [TokenizationToken(raw_token[raw_token_field_name]) for raw_token in response["token_ids"]]
+
+    def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        raise NotImplementedError("The xAI API does not support decoding.")

helm/{proxy/tokenizers → tokenizers}/http_model_tokenizer.py

@@ -1,6 +1,6 @@
 import os
 from dataclasses import asdict
-from typing import Optional
+from typing import Any, Dict, Optional
 
 from helm.common.cache import Cache, CacheConfig
 from helm.common.request import wrap_request_time
@@ -11,7 +11,7 @@ from helm.common.tokenization_request import (
     TokenizationRequestResult,
     TokenizationToken,
 )
-from .tokenizer import Tokenizer
+from helm.tokenizers.tokenizer import Tokenizer
 
 import requests
 
@@ -42,7 +42,7 @@ class HTTPModelTokenizer(Tokenizer):
 
         try:
 
-            def do_it():
+            def do_it() -> Dict[str, Any]:
                 url = f"{self.base_url}/tokenize"
                 response = requests.post(url, json=raw_request)
                 response.raise_for_status()
@@ -70,7 +70,7 @@ class HTTPModelTokenizer(Tokenizer):
 
         try:
 
-            def do_it():
+            def do_it() -> Dict[str, Any]:
                 url = f"{self.base_url}/decode"
                 response = requests.post(url, json={"tokens": request.tokens})
                 response.raise_for_status()

helm/{proxy/tokenizers → tokenizers}/huggingface_tokenizer.py

@@ -1,5 +1,5 @@
 import os
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, cast
 from threading import Lock
 from helm.common.cache import CacheConfig
 from helm.common.concurrency import ThreadSafeWrapper
@@ -7,28 +7,8 @@ from helm.common.concurrency import ThreadSafeWrapper
 from transformers import AutoTokenizer, PreTrainedTokenizerBase
 
 from helm.common.hierarchical_logger import htrack_block, hlog
-from .caching_tokenizer import CachingTokenizer
-from .tokenizer import cleanup_tokens
-
-
-# TODO: Delete this.
-_MODEL_NAME_ALIASES: Dict[str, str] = {
-    "google/t5-11b": "t5-11b",
-    "huggingface/gpt2": "gpt2",
-    "huggingface/santacoder": "bigcode/santacoder",
-    "huggingface/starcoder": "bigcode/starcoder",
-    "writer/gpt2": "gpt2",  # Palmyra models do not support echo
-    # So they have a different TokenizerConfig called "writer/gpt2"
-    # when in reality they use the same tokenizer as "huggingface/gpt2"
-    "microsoft/gpt2": "gpt2",  # Same as above
-}
-"""Mapping of some HELM model names to Hugging Face pretrained model name."""
-
-
-# TODO: Delete this.
-def resolve_alias(model_name: str) -> str:
-    """Resolve some HELM model names to Hugging Face pretrained model name."""
-    return _MODEL_NAME_ALIASES.get(model_name, model_name)
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
+from helm.tokenizers.tokenizer import cleanup_tokens
 
 
 WrappedPreTrainedTokenizer = ThreadSafeWrapper[PreTrainedTokenizerBase]
@@ -49,8 +29,17 @@ class HuggingFaceTokenizer(CachingTokenizer):
     _tokenizers: Dict[str, WrappedPreTrainedTokenizer] = {}
     _tokenizers_lock: Lock = Lock()
 
-    def __init__(self, cache_config: CacheConfig, pretrained_model_name_or_path: Optional[str] = None, **kwargs):
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        tokenizer_name: str,
+        pretrained_model_name_or_path: Optional[str] = None,
+        **kwargs,
+    ):
         super().__init__(cache_config=cache_config)
+        self._helm_tokenizer_name = (
+            tokenizer_name  # HELM tokenizer name (e.g. "huggingface/gpt2"), *not* Hugging Face Hub Model ID
+        )
         self._pretrained_model_name_or_path = pretrained_model_name_or_path
         self._kwargs = kwargs
 
@@ -60,7 +49,10 @@ class HuggingFaceTokenizer(CachingTokenizer):
         # To avoid deadlocks when using HuggingFace tokenizers with multiple processes
         # TODO: Figure out if we actually need this.
         os.environ["TOKENIZERS_PARALLELISM"] = "False"
-
+        from_pretrained_kwargs = {**kwargs}
+        # If unspecified, set `use_fast=True` by default.
+        if "use_fast" not in from_pretrained_kwargs:
+            from_pretrained_kwargs["use_fast"] = True
         try:
             # From the Hugging Face documentation, "local_files_only(defaults to False) —
             # Whether or not to only look at local files".
@@ -73,14 +65,14 @@ class HuggingFaceTokenizer(CachingTokenizer):
             # Tokenizers, which are written in Rust." So, use the "fast" version of the tokenizers if available.
             return WrappedPreTrainedTokenizer(
                 AutoTokenizer.from_pretrained(
-                    pretrained_model_name_or_path, local_files_only=True, use_fast=True, **kwargs
+                    pretrained_model_name_or_path, local_files_only=True, **from_pretrained_kwargs
                 )
             )
         except OSError:
             hlog(f"Local files do not exist for HuggingFace tokenizer: {pretrained_model_name_or_path}. Downloading...")
             return WrappedPreTrainedTokenizer(
                 AutoTokenizer.from_pretrained(
-                    pretrained_model_name_or_path, local_files_only=False, use_fast=True, **kwargs
+                    pretrained_model_name_or_path, local_files_only=False, **from_pretrained_kwargs
                 )
             )
 
@@ -104,31 +96,34 @@ class HuggingFaceTokenizer(CachingTokenizer):
                     )
         return HuggingFaceTokenizer._tokenizers[helm_tokenizer_name]
 
-    def _get_tokenizer_for_request(self, request: Dict[str, Any]) -> WrappedPreTrainedTokenizer:
-        """Method used in both _tokenize_do_it and _decode_do_it to get the tokenizer."""
-        pretrained_model_name_or_path: str
-        if self._pretrained_model_name_or_path:
-            pretrained_model_name_or_path = self._pretrained_model_name_or_path
-        else:
-            pretrained_model_name_or_path = resolve_alias(request["tokenizer"])
+    def get_wrapped_tokenizer(self) -> WrappedPreTrainedTokenizer:
+        """Get the underlying Hugging Face WrappedPreTrainedTokenizer."""
+        pretrained_model_name_or_path = (
+            self._pretrained_model_name_or_path if self._pretrained_model_name_or_path else self._helm_tokenizer_name
+        )
         return HuggingFaceTokenizer.get_tokenizer(
-            helm_tokenizer_name=request["tokenizer"],
+            helm_tokenizer_name=self._helm_tokenizer_name,
             pretrained_model_name_or_path=pretrained_model_name_or_path,
             **self._kwargs,
         )
 
     def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        if request["tokenizer"] != self._helm_tokenizer_name:
+            raise ValueError(
+                f"This HuggingFaceTokenizer expects tokenizer to be {self._helm_tokenizer_name} "
+                "but instead the request has tokenizer {request['tokenizer']}"
+            )
         if request["encode"]:
             if request["truncation"]:
-                with self._get_tokenizer_for_request(request) as tokenizer:
+                with self.get_wrapped_tokenizer() as tokenizer:
                     tokens = tokenizer.encode(
                         request["text"],
                         truncation=request["truncation"],
-                        max_length=request["max_length"],
+                        max_length=max(request["max_length"], 0),
                         add_special_tokens=False,
                     )
             else:
-                with self._get_tokenizer_for_request(request) as tokenizer:
+                with self.get_wrapped_tokenizer() as tokenizer:
                     tokens = tokenizer.encode(request["text"], add_special_tokens=False)
         else:
             if "gpt" in request["tokenizer"] or request["tokenizer"] in [
@@ -140,7 +135,7 @@ class HuggingFaceTokenizer(CachingTokenizer):
                 # convert_tokens_to_string method. We prefer to use this method instead
                 # of the hacky cleanup_tokens method below as it might handle cases
                 # we haven't thought of in cleanup_tokens.
-                with self._get_tokenizer_for_request(request) as tokenizer:
+                with self.get_wrapped_tokenizer() as tokenizer:
                     tokens = [
                         tokenizer.convert_tokens_to_string([token]) for token in tokenizer.tokenize(request["text"])
                     ]
@@ -153,13 +148,21 @@ class HuggingFaceTokenizer(CachingTokenizer):
                 # But this replaces all the "▁" characters by "", which is not what we want.
                 # This would be problematic as tokenize(" Hello", encode=False) would return ["Hello"]
                 # Just like tokenize("Hello", encode=False) would return ["Hello"].
-                with self._get_tokenizer_for_request(request) as tokenizer:
+                with self.get_wrapped_tokenizer() as tokenizer:
                     tokens = tokenizer.tokenize(request["text"])
+                # Some tokenizers (e.g. Qwen/Qwen-7B) return the tokens as bytes, so we have to decode them to strings.
+                if tokens and type(tokens[0]) == bytes:
+                    tokens = [cast(bytes, token).decode(errors="ignore") for token in tokens]
                 tokens = cleanup_tokens(tokens, request["tokenizer"])
         return {"tokens": tokens}
 
     def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
-        with self._get_tokenizer_for_request(request) as tokenizer:
+        if request["tokenizer"] != self._helm_tokenizer_name:
+            raise ValueError(
+                f"This HuggingFaceTokenizer expects tokenizer to be {self._helm_tokenizer_name} "
+                "but instead the request has tokenizer {request['tokenizer']}"
+            )
+        with self.get_wrapped_tokenizer() as tokenizer:
             text = tokenizer.decode(
                 request["tokens"], clean_up_tokenization_spaces=request["clean_up_tokenization_spaces"]
             )