crfm-helm 0.5.2__py3-none-any.whl → 0.5.3__py3-none-any.whl

helm/config/tokenizer_configs.yaml

@@ -17,11 +17,31 @@ tokenizer_configs:
     prefix_token: "<s>"
 
   # AI21
-  - name: ai21/j1
+  - name: ai21/j2-tokenizer
     tokenizer_spec:
-      class_name: "helm.tokenizers.ai21_tokenizer.AI21Tokenizer"
-    end_of_text_token: " "
-    prefix_token: ""
+      class_name: "helm.tokenizers.ai21_tokenizer.AI21LocalTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|startoftext|>"
+  - name: ai21/jamba-tokenizer
+    tokenizer_spec:
+      class_name: "helm.tokenizers.ai21_tokenizer.AI21LocalTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|startoftext|>"
+  - name: ai21/jamba-instruct-tokenizer
+    tokenizer_spec:
+      class_name: "helm.tokenizers.ai21_tokenizer.AI21LocalTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|startoftext|>"
+  - name: ai21/jamba-1.5-mini-tokenizer
+    tokenizer_spec:
+      class_name: "helm.tokenizers.ai21_tokenizer.AI21LocalTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|startoftext|>"
+  - name: ai21/jamba-1.5-large-tokenizer
+    tokenizer_spec:
+      class_name: "helm.tokenizers.ai21_tokenizer.AI21LocalTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|startoftext|>"
 
   # AlephAlpha
   - name: AlephAlpha/luminous-base
@@ -45,6 +65,24 @@ tokenizer_configs:
     end_of_text_token: ""
     prefix_token: ""
 
+  # Alibaba DAMO Academy
+
+  - name: damo/seallm-7b-v2
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: SeaLLMs/SeaLLM-7B-v2
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+  - name: damo/seallm-7b-v2.5
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: SeaLLMs/SeaLLM-7B-v2.5
+    end_of_text_token: "<eos>"
+    prefix_token: "<bos>"
+
   # Anthropic
   - name: anthropic/claude
     tokenizer_spec:
@@ -77,12 +115,6 @@ tokenizer_configs:
     prefix_token: ""
 
   # Cohere
-  - name: cohere/cohere
-    tokenizer_spec:
-      class_name: "helm.tokenizers.cohere_tokenizer.CohereTokenizer"
-    end_of_text_token: ""
-    prefix_token: ":"
-
   - name: cohere/command
     tokenizer_spec:
       class_name: "helm.tokenizers.cohere_tokenizer.CohereLocalTokenizer"
@@ -199,6 +231,11 @@ tokenizer_configs:
       class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "<eos>"
     prefix_token: "<bos>"
+  - name: google/gemma-2-9b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<eos>"
+    prefix_token: "<bos>"
 
   # Hf-internal-testing
 
@@ -280,6 +317,14 @@ tokenizer_configs:
     prefix_token: "<|begin_of_text|>"
     end_of_text_token: "<|end_of_text|>"
 
+  - name: meta/llama-3.1-8b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: meta-llama/Meta-Llama-3.1-8B-Instruct
+    prefix_token: "<|begin_of_text|>"
+    end_of_text_token: "<|end_of_text|>"
+
   # 01-ai
   - name: 01-ai/Yi-6B
     tokenizer_spec:
@@ -324,6 +369,20 @@ tokenizer_configs:
     end_of_text_token: "<|endoftext|>"  
     prefix_token: "<|endoftext|>"
 
+  - name: microsoft/phi-3-small-8k-instruct
+    tokenizer_spec:  
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        trust_remote_code: true
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  - name: microsoft/phi-3-medium-4k-instruct
+    tokenizer_spec:  
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<s>"
+
   # Mistralai
   - name: mistralai/Mistral-7B-v0.1
     tokenizer_spec:
@@ -349,6 +408,18 @@ tokenizer_configs:
     end_of_text_token: "</s>"
     prefix_token: "<s>"
 
+  - name: mistralai/Mistral-Nemo-Base-2407
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+  - name: mistralai/Mistral-Large-Instruct-2407
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
   # Neurips
   - name: neurips/local
     tokenizer_spec:
@@ -356,7 +427,17 @@ tokenizer_configs:
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
 
-  # Openai
+  # NVIDIA
+  - name: nvidia/nemotron-4-340b-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Xenova/Nemotron-4-340B-Instruct-Tokenizer
+        revision: b7aa0de92cda9f9e722d58d6ca90f46ae17d4701
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  # OpenAI
   - name: openai/cl100k_base
     tokenizer_spec:
       class_name: "helm.tokenizers.tiktoken_tokenizer.TiktokenTokenizer"
@@ -375,6 +456,14 @@ tokenizer_configs:
     end_of_text_token: ""
     prefix_token: ""
 
+  # OpenThaiGPT
+  - name: openthaigpt/openthaigpt-1.0.0-7b-chat
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+  # Qwen
   - name: qwen/qwen-7b
     tokenizer_spec:
       class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
@@ -420,6 +509,15 @@ tokenizer_configs:
     end_of_text_token: "<|endoftext|>"
     prefix_token: ""
 
+  # SambaLingo
+  - name: sambanova/sambalingo-thai-base
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
   # Snowflake
   - name: snowflake/snowflake-arctic-instruct
     tokenizer_spec:
@@ -444,6 +542,13 @@ tokenizer_configs:
     end_of_text_token: "</s>"
     prefix_token: ""
 
+  # Typhoon
+  - name: scb10x/typhoon-7b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
   # Writer
   - name: writer/gpt2
     tokenizer_spec:

helm/proxy/example_queries.py

@@ -22,7 +22,6 @@ example_queries = [
             temperature: 0.5  # Medium amount of randomness
             stop_sequences: [.]  # Stop when you hit a period
             model: openai/gpt-3.5-turbo-0613
-            model_deployment: openai/gpt-3.5-turbo-0613
             """
         ),
         environments="",
@@ -35,24 +34,24 @@ example_queries = [
             stop_sequences: [\\n]  # Stop when you hit a newline
             num_completions: 5  # Generate many samples
             model: openai/gpt-3.5-turbo-0613
-            model_deployment: openai/gpt-3.5-turbo-0613
             """
         ),
         environments="",
     ),
-    Query(
-        prompt="The quick brown fox jumps over the lazy dog.",
-        settings=dedent(
-            """
-            echo_prompt: true  # Analyze the prompt
-            max_tokens: 0  # Don't generate any more
-            top_k_per_token: 5  # Show alternatives for each position
-            model: openai/davinci-002
-            model_deployment: openai/davinci-002
-            """
-        ),
-        environments=dedent(""),
-    ),
+    # Disabled because `max_tokens: 0` no longer works on the OpenAI API
+    # Query(
+    #     prompt="The quick brown fox jumps over the lazy dog.",
+    #     settings=dedent(
+    #         """
+    #         echo_prompt: true  # Analyze the prompt
+    #         max_tokens: 0  # Don't generate any more
+    #         top_k_per_token: 5  # Show alternatives for each position
+    #         model: openai/text-davinci-002
+    #         model_deployment: openai/text-davinci-002
+    #         """
+    #     ),
+    #     environments=dedent(""),
+    # ),
     Query(
         prompt="Odd numbers: 1 -> 3 -> 5",
         settings=dedent(
@@ -60,7 +59,6 @@ example_queries = [
             temperature: 0  # Deterministic
             max_tokens: 50
             model: openai/gpt-3.5-turbo-0613
-            model_deployment: openai/gpt-3.5-turbo-0613
             """
         ),
         environments="",
@@ -73,7 +71,6 @@ example_queries = [
             stop_sequences: [.]
             # Try out multiple models
             model: ${model}
-            model_deployment: ${model}
             """
         ),
         environments=dedent(
@@ -100,7 +97,6 @@ example_queries = [
             num_completions: 5
             # Try out multiple models
             model: ${model}
-            model_deployment: ${model}
             """
         ),
         environments=dedent(
@@ -136,7 +132,6 @@ example_queries = [
             top_k_per_token: 4
             # Try out multiple models
             model: ${model}
-            model_deployment: ${model}
             """
         ),
         environments=dedent(
@@ -150,7 +145,6 @@ example_queries = [
         settings=dedent(
             """
             model: openai/gpt-3.5-turbo-0613
-            model_deployment: openai/gpt-3.5-turbo-0613
             """
         ),
         environments="",
@@ -163,7 +157,6 @@ example_queries = [
             stop_sequences: [\\n]
             # Try out multiple models
             model: ${model}
-            model_deployment: ${model}
             """
         ),
         environments=dedent(

helm/proxy/services/server_service.py

@@ -25,7 +25,6 @@ from helm.common.hierarchical_logger import hlog
 from helm.proxy.accounts import Accounts, Account
 from helm.clients.auto_client import AutoClient
 from helm.clients.moderation_api_client import ModerationAPIClient
-from helm.clients.perspective_api_client import PerspectiveAPIClient
 from helm.clients.image_generation.nudity_check_client import NudityCheckClient
 from helm.clients.gcs_client import GCSClient
 from helm.clients.clip_score_client import CLIPScoreClient
@@ -75,7 +74,7 @@ class ServerService(Service):
         # Lazily instantiate the following clients
         self.moderation_api_client: Optional[ModerationAPIClient] = None
         self.toxicity_classifier_client: Optional[ToxicityClassifierClient] = None
-        self.perspective_api_client: Optional[PerspectiveAPIClient] = None
+        self.perspective_api_client: Optional[ToxicityClassifierClient] = None
         self.nudity_check_client: Optional[NudityCheckClient] = None
         self.clip_score_client: Optional[CLIPScoreClient] = None
         self.gcs_client: Optional[GCSClient] = None

helm/proxy/token_counters/test_auto_token_counter.py

@@ -13,8 +13,8 @@ class TestAutoTokenCounter:
         )
         # The following prompt has 51 tokens according to the GPT-2 tokenizer
         request = Request(
-            model="openai/text-davinci-002",
-            model_deployment="openai/text-davinci-002",
+            model="openai/gpt2",
+            model_deployment="huggingface/gpt2",
             prompt="The Center for Research on Foundation Models (CRFM) is "
             "an interdisciplinary initiative born out of the Stanford "
             "Institute for Human-Centered Artificial Intelligence (HAI) "

helm/tokenizers/ai21_tokenizer.py

@@ -1,60 +1,52 @@
-from typing import Any, Dict, List
-import requests
-
-from dacite import from_dict
-
-from helm.common.cache import Cache, CacheConfig
-from helm.common.tokenization_request import (
-    TokenizationRequest,
-    TokenizationRequestResult,
-    TokenizationToken,
-    TextRange,
-    DecodeRequest,
-    DecodeRequestResult,
-)
-from helm.clients.ai21_utils import AI21RequestError, handle_failed_request
-from .tokenizer import Tokenizer
-
-
-class AI21Tokenizer(Tokenizer):
-    def __init__(self, api_key: str, cache_config: CacheConfig) -> None:
-        self.cache = Cache(cache_config)
-        self.api_key: str = api_key
-
-    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
-        """
-        Tokenizes the text by using the AI21 endpoint: https://api.ai21.com/studio/v1/tokenize.
-        """
-        # TODO: Does not support encoding
-        raw_request: Dict[str, str] = {"text": request.text}
-
-        def do_it() -> Dict[str, Any]:
-            response = requests.post(
-                "https://api.ai21.com/studio/v1/tokenize",
-                headers={"Authorization": f"Bearer {self.api_key}"},
-                json=raw_request,
-            ).json()
-
-            # If 'tokens' is not present in the response, assume request failed.
-            if "tokens" not in response:
-                handle_failed_request(api_type="tokenizer", response=response)
-
-            return response
-
-        try:
-            response, cached = self.cache.get(raw_request, do_it)
-        except AI21RequestError:
-            return TokenizationRequestResult(success=False, cached=False, text="", tokens=[])
-
-        # Each token is represented like this in the response:
-        # {'token': '▁Hello', 'textRange': {'start': 0, 'end': 5}}
-        tokens: List[TokenizationToken] = []
-        for token_dict in response["tokens"]:
-            tokens.append(
-                TokenizationToken(value=token_dict["token"], text_range=from_dict(TextRange, token_dict["textRange"]))
+import threading
+from typing import Any, Dict
+
+from helm.common.cache import CacheConfig
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
+
+try:
+    from ai21_tokenizer import Tokenizer as SDKTokenizer
+    from ai21_tokenizer.base_tokenizer import BaseTokenizer
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["ai21"])
+
+
+class AI21LocalTokenizer(CachingTokenizer):
+    """AI21 tokenizer using the AI21 Python library."""
+
+    def __init__(self, cache_config: CacheConfig) -> None:
+        super().__init__(cache_config)
+        self._tokenizers_lock = threading.Lock()
+        self.tokenizers: Dict[str, BaseTokenizer] = {}
+
+    def _get_tokenizer(self, tokenizer_name: str) -> BaseTokenizer:
+        with self._tokenizers_lock:
+            if tokenizer_name not in self.tokenizers:
+                self.tokenizers[tokenizer_name] = SDKTokenizer.get_tokenizer(tokenizer_name)
+            return self.tokenizers[tokenizer_name]
+
+    def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        tokenizer_name = request["tokenizer"].split("/")[1]
+        tokenizer = self._get_tokenizer(tokenizer_name)
+        if request["truncation"]:
+            token_ids = tokenizer.encode(
+                text=request["text"],
+                truncation=request["truncation"],
+                max_length=request["max_length"],
+                add_special_tokens=False,
             )
-        text: str = response["text"]
-        return TokenizationRequestResult(success=True, cached=cached, tokens=tokens, text=text)
-
-    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
-        raise NotImplementedError("Not supported")
+        else:
+            token_ids = tokenizer.encode(
+                text=request["text"],
+                add_special_tokens=False,
+            )
+        if request["encode"]:
+            return {"tokens": token_ids}
+        else:
+            return {"tokens": tokenizer.convert_ids_to_tokens(token_ids)}
+
+    def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        tokenizer_name = request["tokenizer"].split("/")[1]
+        tokenizer = self._get_tokenizer(tokenizer_name)
+        return {"text": tokenizer.decode(request["tokens"])}

helm/tokenizers/cohere_tokenizer.py

@@ -1,5 +1,3 @@
-import json
-import requests
 from typing import Any, Dict, List, Optional
 
 import cohere
@@ -8,84 +6,11 @@ from cohere.manually_maintained.tokenizers import get_hf_tokenizer
 from helm.common.cache import CacheConfig
 from helm.common.tokenization_request import (
     TokenizationRequest,
-    DecodeRequest,
-    DecodeRequestResult,
     TokenizationToken,
 )
-from helm.clients.cohere_utils import get_cohere_url, DEFAULT_COHERE_API_VERSION
 from helm.tokenizers.caching_tokenizer import CachingTokenizer
 
 
-class CohereTokenizer(CachingTokenizer):
-    # From "https://docs.cohere.ai/versioning-reference",
-    # "this version [2021-11-08] introduces multiple generations, meaning that the generations endpoint will
-    # now accept a num_generations argument in the JSON and will always return an array of generations"
-    # Note that the API version is decoupled from the model version.
-    DEFAULT_API_VERSION: str = "2021-11-08"
-
-    TOKENIZE_ENDPOINT: str = "tokenize"
-
-    # According to https://docs.cohere.ai/tokenize-reference#request, for tokenize, text: "the string to
-    # be tokenized, the minimum text length is 1 character, and the maximum text length is 65536 characters."
-    # However, even sending a request with 60,000 characters sometimes fails, so we set the
-    # maximum length to 50,000, which is about 8,333 tokens.
-    # TODO: followed up with Cohere support with an example of a failure case
-    TOKENIZE_API_MAX_TEXT_LENGTH: int = 50_000
-
-    def __init__(self, api_key: str, cache_config: CacheConfig) -> None:
-        super().__init__(cache_config)
-        self.api_key: str = api_key
-
-    def _tokenization_request_to_cache_key(self, request: TokenizationRequest) -> Dict[str, Any]:
-        # This cache key is used to preserve our existing Cache (10/17/2023)
-        return {"text": request.text}
-
-    def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
-        """
-        Send the request to the Cohere Tokenize API.
-
-        From https://docs.cohere.ai/tokenize-reference, for text "tokenize me! :D", the response will be:
-
-        {
-            "tokens": [34160, 974, 514, 34, 1420, 69]
-            "token_strings": ["token", "ize", " me", "!", " :", "D"]
-        }
-        """
-        text: str = request["text"]
-        assert (
-            1 <= len(text) <= CohereTokenizer.TOKENIZE_API_MAX_TEXT_LENGTH
-        ), f"Invalid text length: {len(text)}. Valid length: [1..{CohereTokenizer.TOKENIZE_API_MAX_TEXT_LENGTH:,d}]"
-
-        response = requests.request(
-            method="POST",
-            url=get_cohere_url(CohereTokenizer.TOKENIZE_ENDPOINT),
-            headers={
-                "Authorization": f"BEARER {self.api_key}",
-                "Content-Type": "application/json",
-                "Cohere-Version": DEFAULT_COHERE_API_VERSION,
-            },
-            data=json.dumps(request),
-        )
-        result = json.loads(response.text)
-        assert "message" not in result.keys(), f"Request failed with error {result['message']}"
-        assert "tokens" in result and "token_strings" in result, f"Invalid response: {result}"
-        # This output format is used to preserve our existing Cache (10/17/2023)
-        return result
-
-    def _tokenization_raw_response_to_tokens(
-        self, response: Dict[str, Any], request: TokenizationRequest
-    ) -> List[TokenizationToken]:
-        tokens = response["tokens" if request.encode else "token_strings"]
-        return [TokenizationToken(token) for token in tokens]
-
-    def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
-        # Defined for mypy but decode() already raises NotImplementedError
-        raise NotImplementedError("The Cohere API does not support decoding.")
-
-    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
-        raise NotImplementedError("The Cohere API does not support decoding.")
-
-
 class CohereLocalTokenizer(CachingTokenizer):
     """Cohere tokenizer using the Cohere Python library."""
 

helm/tokenizers/huggingface_tokenizer.py

@@ -53,7 +53,6 @@ class HuggingFaceTokenizer(CachingTokenizer):
         # If unspecified, set `use_fast=True` by default.
         if "use_fast" not in from_pretrained_kwargs:
             from_pretrained_kwargs["use_fast"] = True
-        print(from_pretrained_kwargs)
         try:
             # From the Hugging Face documentation, "local_files_only(defaults to False) —
             # Whether or not to only look at local files".

helm/tokenizers/test_ai21_tokenizer.py

@@ -0,0 +1,48 @@
+import pytest
+
+from helm.common.cache import BlackHoleCacheConfig
+from helm.common.tokenization_request import (
+    DecodeRequest,
+    TokenizationRequest,
+    TokenizationToken,
+)
+
+
+@pytest.mark.models
+def test_tokenize():
+    from helm.tokenizers.ai21_tokenizer import AI21LocalTokenizer
+
+    tokenizer = AI21LocalTokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="ai21/jamba-instruct-tokenizer", text="otter 🦦")
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [
+        TokenizationToken(token) for token in ["ot", "ter", "▁", "<0xF0>", "<0x9F>", "<0xA6>", "<0xA6>"]
+    ]
+
+
+@pytest.mark.models
+def test_encode():
+    from helm.tokenizers.ai21_tokenizer import AI21LocalTokenizer
+
+    tokenizer = AI21LocalTokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="ai21/jamba-instruct-tokenizer", text="otter 🦦", encode=True)
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in [1860, 1901, 62934, 1784, 1703, 1710, 1710]]
+
+
+@pytest.mark.models
+def test_decode():
+    from helm.tokenizers.ai21_tokenizer import AI21LocalTokenizer
+
+    tokenizer = AI21LocalTokenizer(cache_config=BlackHoleCacheConfig())
+    request = DecodeRequest(
+        tokenizer="ai21/jamba-instruct-tokenizer", tokens=[1860, 1901, 62934, 1784, 1703, 1710, 1710]
+    )
+    result = tokenizer.decode(request)
+    assert result.success
+    assert not result.cached
+    assert result.text == "otter 🦦"