crfm-helm 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

helm/tokenizers/cohere_tokenizer.py

@@ -1,83 +1,50 @@
-import json
-import requests
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
+
+import cohere
+from cohere.manually_maintained.tokenizers import get_hf_tokenizer
 
 from helm.common.cache import CacheConfig
 from helm.common.tokenization_request import (
     TokenizationRequest,
-    DecodeRequest,
-    DecodeRequestResult,
     TokenizationToken,
 )
-from helm.clients.cohere_utils import get_cohere_url, DEFAULT_COHERE_API_VERSION
-from .caching_tokenizer import CachingTokenizer
-
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
 
-class CohereTokenizer(CachingTokenizer):
-    # From "https://docs.cohere.ai/versioning-reference",
-    # "this version [2021-11-08] introduces multiple generations, meaning that the generations endpoint will
-    # now accept a num_generations argument in the JSON and will always return an array of generations"
-    # Note that the API version is decoupled from the model version.
-    DEFAULT_API_VERSION: str = "2021-11-08"
 
-    TOKENIZE_ENDPOINT: str = "tokenize"
+class CohereLocalTokenizer(CachingTokenizer):
+    """Cohere tokenizer using the Cohere Python library."""
 
-    # According to https://docs.cohere.ai/tokenize-reference#request, for tokenize, text: "the string to
-    # be tokenized, the minimum text length is 1 character, and the maximum text length is 65536 characters."
-    # However, even sending a request with 60,000 characters sometimes fails, so we set the
-    # maximum length to 50,000, which is about 8,333 tokens.
-    # TODO: followed up with Cohere support with an example of a failure case
-    TOKENIZE_API_MAX_TEXT_LENGTH: int = 50_000
-
-    def __init__(self, api_key: str, cache_config: CacheConfig) -> None:
+    def __init__(self, api_key: Optional[str], cache_config: CacheConfig) -> None:
         super().__init__(cache_config)
-        self.api_key: str = api_key
+        self.client = cohere.Client(api_key)
 
     def _tokenization_request_to_cache_key(self, request: TokenizationRequest) -> Dict[str, Any]:
-        # This cache key is used to preserve our existing Cache (10/17/2023)
-        return {"text": request.text}
+        return {"text": request.text, "tokenizer": request.tokenizer}
 
     def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
-        """
-        Send the request to the Cohere Tokenize API.
-
-        From https://docs.cohere.ai/tokenize-reference, for text "tokenize me! :D", the response will be:
-
-        {
-            "tokens": [34160, 974, 514, 34, 1420, 69]
-            "token_strings": ["token", "ize", " me", "!", " :", "D"]
-        }
-        """
-        text: str = request["text"]
-        assert (
-            1 <= len(text) <= CohereTokenizer.TOKENIZE_API_MAX_TEXT_LENGTH
-        ), f"Invalid text length: {len(text)}. Valid length: [1..{CohereTokenizer.TOKENIZE_API_MAX_TEXT_LENGTH:,d}]"
-
-        response = requests.request(
-            method="POST",
-            url=get_cohere_url(CohereTokenizer.TOKENIZE_ENDPOINT),
-            headers={
-                "Authorization": f"BEARER {self.api_key}",
-                "Content-Type": "application/json",
-                "Cohere-Version": DEFAULT_COHERE_API_VERSION,
-            },
-            data=json.dumps(request),
+        model: str = request["tokenizer"].split("/")[1]
+        # Workaround for https://github.com/cohere-ai/cohere-python/issues/493
+        # `token_strings` are always set to `[]`, so we have to populate it ourselves.
+        response = self.client.tokenize(text=request["text"], model=model)
+        response_dict = response.dict()
+        response_dict["token_strings"] = get_hf_tokenizer(self.client, model).decode_batch(
+            [[token] for token in response.tokens]
         )
-        result = json.loads(response.text)
-        assert "message" not in result.keys(), f"Request failed with error {result['message']}"
-        assert "tokens" in result and "token_strings" in result, f"Invalid response: {result}"
-        # This output format is used to preserve our existing Cache (10/17/2023)
-        return result
+        return response_dict
 
     def _tokenization_raw_response_to_tokens(
         self, response: Dict[str, Any], request: TokenizationRequest
     ) -> List[TokenizationToken]:
-        tokens = response["tokens" if request.encode else "token_strings"]
-        return [TokenizationToken(token) for token in tokens]
+        tokens: List[TokenizationToken] = []
+        if request.encode:
+            tokens = [TokenizationToken(token) for token in response["tokens"]]
+        else:
+            tokens = [TokenizationToken(token) for token in response["token_strings"]]
+        if request.truncation:
+            tokens = tokens[: request.max_length]
+        return tokens
 
     def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
-        # Defined for mypy but decode() already raises NotImplementedError
-        raise NotImplementedError("The Cohere API does not support decoding.")
-
-    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
-        raise NotImplementedError("The Cohere API does not support decoding.")
+        model: str = request["tokenizer"].split("/")[1]
+        response = self.client.detokenize(tokens=request["tokens"], model=model)
+        return response.dict()

helm/tokenizers/huggingface_tokenizer.py

@@ -29,8 +29,17 @@ class HuggingFaceTokenizer(CachingTokenizer):
     _tokenizers: Dict[str, WrappedPreTrainedTokenizer] = {}
     _tokenizers_lock: Lock = Lock()
 
-    def __init__(self, cache_config: CacheConfig, pretrained_model_name_or_path: Optional[str] = None, **kwargs):
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        tokenizer_name: str,
+        pretrained_model_name_or_path: Optional[str] = None,
+        **kwargs,
+    ):
         super().__init__(cache_config=cache_config)
+        self._helm_tokenizer_name = (
+            tokenizer_name  # HELM tokenizer name (e.g. "huggingface/gpt2"), *not* Hugging Face Hub Model ID
+        )
         self._pretrained_model_name_or_path = pretrained_model_name_or_path
         self._kwargs = kwargs
 
@@ -40,7 +49,10 @@ class HuggingFaceTokenizer(CachingTokenizer):
         # To avoid deadlocks when using HuggingFace tokenizers with multiple processes
         # TODO: Figure out if we actually need this.
         os.environ["TOKENIZERS_PARALLELISM"] = "False"
-
+        from_pretrained_kwargs = {**kwargs}
+        # If unspecified, set `use_fast=True` by default.
+        if "use_fast" not in from_pretrained_kwargs:
+            from_pretrained_kwargs["use_fast"] = True
         try:
             # From the Hugging Face documentation, "local_files_only(defaults to False) —
             # Whether or not to only look at local files".
@@ -53,14 +65,14 @@ class HuggingFaceTokenizer(CachingTokenizer):
             # Tokenizers, which are written in Rust." So, use the "fast" version of the tokenizers if available.
             return WrappedPreTrainedTokenizer(
                 AutoTokenizer.from_pretrained(
-                    pretrained_model_name_or_path, local_files_only=True, use_fast=True, **kwargs
+                    pretrained_model_name_or_path, local_files_only=True, **from_pretrained_kwargs
                 )
             )
         except OSError:
             hlog(f"Local files do not exist for HuggingFace tokenizer: {pretrained_model_name_or_path}. Downloading...")
             return WrappedPreTrainedTokenizer(
                 AutoTokenizer.from_pretrained(
-                    pretrained_model_name_or_path, local_files_only=False, use_fast=True, **kwargs
+                    pretrained_model_name_or_path, local_files_only=False, **from_pretrained_kwargs
                 )
             )
 
@@ -84,21 +96,26 @@ class HuggingFaceTokenizer(CachingTokenizer):
                     )
         return HuggingFaceTokenizer._tokenizers[helm_tokenizer_name]
 
-    def _get_tokenizer_for_request(self, request: Dict[str, Any]) -> WrappedPreTrainedTokenizer:
-        """Method used in both _tokenize_do_it and _decode_do_it to get the tokenizer."""
+    def get_wrapped_tokenizer(self) -> WrappedPreTrainedTokenizer:
+        """Get the underlying Hugging Face WrappedPreTrainedTokenizer."""
         pretrained_model_name_or_path = (
-            self._pretrained_model_name_or_path if self._pretrained_model_name_or_path else request["tokenizer"]
+            self._pretrained_model_name_or_path if self._pretrained_model_name_or_path else self._helm_tokenizer_name
         )
         return HuggingFaceTokenizer.get_tokenizer(
-            helm_tokenizer_name=request["tokenizer"],
+            helm_tokenizer_name=self._helm_tokenizer_name,
             pretrained_model_name_or_path=pretrained_model_name_or_path,
             **self._kwargs,
         )
 
     def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        if request["tokenizer"] != self._helm_tokenizer_name:
+            raise ValueError(
+                f"This HuggingFaceTokenizer expects tokenizer to be {self._helm_tokenizer_name} "
+                "but instead the request has tokenizer {request['tokenizer']}"
+            )
         if request["encode"]:
             if request["truncation"]:
-                with self._get_tokenizer_for_request(request) as tokenizer:
+                with self.get_wrapped_tokenizer() as tokenizer:
                     tokens = tokenizer.encode(
                         request["text"],
                         truncation=request["truncation"],
@@ -106,7 +123,7 @@ class HuggingFaceTokenizer(CachingTokenizer):
                         add_special_tokens=False,
                     )
             else:
-                with self._get_tokenizer_for_request(request) as tokenizer:
+                with self.get_wrapped_tokenizer() as tokenizer:
                     tokens = tokenizer.encode(request["text"], add_special_tokens=False)
         else:
             if "gpt" in request["tokenizer"] or request["tokenizer"] in [
@@ -118,7 +135,7 @@ class HuggingFaceTokenizer(CachingTokenizer):
                 # convert_tokens_to_string method. We prefer to use this method instead
                 # of the hacky cleanup_tokens method below as it might handle cases
                 # we haven't thought of in cleanup_tokens.
-                with self._get_tokenizer_for_request(request) as tokenizer:
+                with self.get_wrapped_tokenizer() as tokenizer:
                     tokens = [
                         tokenizer.convert_tokens_to_string([token]) for token in tokenizer.tokenize(request["text"])
                     ]
@@ -131,7 +148,7 @@ class HuggingFaceTokenizer(CachingTokenizer):
                 # But this replaces all the "▁" characters by "", which is not what we want.
                 # This would be problematic as tokenize(" Hello", encode=False) would return ["Hello"]
                 # Just like tokenize("Hello", encode=False) would return ["Hello"].
-                with self._get_tokenizer_for_request(request) as tokenizer:
+                with self.get_wrapped_tokenizer() as tokenizer:
                     tokens = tokenizer.tokenize(request["text"])
                 # Some tokenizers (e.g. Qwen/Qwen-7B) return the tokens as bytes, so we have to decode them to strings.
                 if tokens and type(tokens[0]) == bytes:
@@ -140,7 +157,12 @@ class HuggingFaceTokenizer(CachingTokenizer):
         return {"tokens": tokens}
 
     def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
-        with self._get_tokenizer_for_request(request) as tokenizer:
+        if request["tokenizer"] != self._helm_tokenizer_name:
+            raise ValueError(
+                f"This HuggingFaceTokenizer expects tokenizer to be {self._helm_tokenizer_name} "
+                "but instead the request has tokenizer {request['tokenizer']}"
+            )
+        with self.get_wrapped_tokenizer() as tokenizer:
             text = tokenizer.decode(
                 request["tokens"], clean_up_tokenization_spaces=request["clean_up_tokenization_spaces"]
             )

helm/tokenizers/test_ai21_tokenizer.py

@@ -0,0 +1,48 @@
+import pytest
+
+from helm.common.cache import BlackHoleCacheConfig
+from helm.common.tokenization_request import (
+    DecodeRequest,
+    TokenizationRequest,
+    TokenizationToken,
+)
+
+
+@pytest.mark.models
+def test_tokenize():
+    from helm.tokenizers.ai21_tokenizer import AI21LocalTokenizer
+
+    tokenizer = AI21LocalTokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="ai21/jamba-instruct-tokenizer", text="otter 🦦")
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [
+        TokenizationToken(token) for token in ["ot", "ter", "▁", "<0xF0>", "<0x9F>", "<0xA6>", "<0xA6>"]
+    ]
+
+
+@pytest.mark.models
+def test_encode():
+    from helm.tokenizers.ai21_tokenizer import AI21LocalTokenizer
+
+    tokenizer = AI21LocalTokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="ai21/jamba-instruct-tokenizer", text="otter 🦦", encode=True)
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in [1860, 1901, 62934, 1784, 1703, 1710, 1710]]
+
+
+@pytest.mark.models
+def test_decode():
+    from helm.tokenizers.ai21_tokenizer import AI21LocalTokenizer
+
+    tokenizer = AI21LocalTokenizer(cache_config=BlackHoleCacheConfig())
+    request = DecodeRequest(
+        tokenizer="ai21/jamba-instruct-tokenizer", tokens=[1860, 1901, 62934, 1784, 1703, 1710, 1710]
+    )
+    result = tokenizer.decode(request)
+    assert result.success
+    assert not result.cached
+    assert result.text == "otter 🦦"

helm/tokenizers/test_cohere_tokenizer.py

@@ -0,0 +1,39 @@
+import pytest
+
+from helm.common.cache import BlackHoleCacheConfig
+from helm.common.tokenization_request import (
+    DecodeRequest,
+    TokenizationRequest,
+    TokenizationToken,
+)
+from helm.tokenizers.cohere_tokenizer import CohereLocalTokenizer
+
+
+@pytest.mark.models
+def test_tokenize():
+    tokenizer = CohereLocalTokenizer(api_key=None, cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="cohere/command", text="otter 🦦")
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in ["ot", "ter", " �", "�", "�"]]
+
+
+@pytest.mark.models
+def test_encode():
+    tokenizer = CohereLocalTokenizer(api_key=None, cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="cohere/command", text="otter 🦦", encode=True)
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in [1741, 1779, 7728, 107, 107]]
+
+
+@pytest.mark.models
+def test_decode():
+    tokenizer = CohereLocalTokenizer(api_key=None, cache_config=BlackHoleCacheConfig())
+    request = DecodeRequest(tokenizer="cohere/command", tokens=[1741, 1779, 7728, 107, 107])
+    result = tokenizer.decode(request)
+    assert result.success
+    assert not result.cached
+    assert result.text == "otter 🦦"

helm/tokenizers/test_huggingface_tokenizer.py

@@ -17,7 +17,11 @@ class TestHuggingFaceGPT2Tokenizer:
     def setup_method(self, method):
         cache_file = tempfile.NamedTemporaryFile(delete=False)
         self.cache_path: str = cache_file.name
-        self.tokenizer = HuggingFaceTokenizer(SqliteCacheConfig(self.cache_path))
+        self.tokenizer = HuggingFaceTokenizer(
+            SqliteCacheConfig(self.cache_path),
+            tokenizer_name="huggingface/gpt2",
+            pretrained_model_name_or_path="openai-community/gpt2",
+        )
 
     def teardown_method(self, method):
         os.remove(self.cache_path)

helm/benchmark/static/benchmarking.css

@@ -1,156 +0,0 @@
-.correct {
-  background-color: #dfffdf;
-}
-
-.wrong {
-  background-color: #ffdfdf;
-}
-
-.scenario-info {
-  margin-top: 30px;
-  margin-bottom: 30px;
-}
-
-td {
-  padding-left: 15px;
-  padding-right: 15px;
-  padding-top: 5px;
-  padding-bottom: 5px;
-  word-wrap: break-word;
-  max-width: 900px;
-  vertical-align: top;
-}
-
-.results-table {
-}
-
-.table-container {
-  margin-top: 30px;
-  margin-bottom: 30px;
-}
-
-tr {
-  border: solid;
-  border-color: #f0f0f0;
-  border-width: 1px 0;
-}
-
-.results-table thead tr {
-  background-color: #f9f9f9;
-}
-
-.logprob {
-  font-size: 8pt;
-  font-style: italic;
-  color: gray;
-}
-
-.list-header {
-  font-size: 24px;
-  font-weight: bold;
-}
-.list-item {
-  color: black;
-  font-size: 14px;
-  white-space: nowrap;
-}
-.list-item-todo {
-  color: lightgray;
-}
-.list-item:hover {
-  color: black;
-  text-decoration: none;
-  background-color: lightgray;
-}
-
-.main-link {
-  color: white;
-  background-color: #53A0C0;
-}
-.main-link:hover {
-  color: lightgray;
-}
-
-.access-open {
-  background-color: lightgreen;
-  width: 100px;
-}
-.access-limited {
-  background-color: yellow;
-  width: 100px;
-}
-.access-restricted {
-  background-color: orange;
-  width: 100px;
-}
-.access-closed {
-  background-color: lightgray;
-  width: 100px;
-}
-
-.technical-details {
-  font-size: 10px;
-  font-style: italic;
-  color: gray;
-}
-
-.logo-container {
-  display: flex;
-  flex-flow: row wrap;
-  justify-content: space-between;
-  padding: 20px;
-}
-
-.logo-item {
-  margin: auto;
-  padding: 10px;
-}
-
-.instance-input {
-  font-style: italic;
-  background-color: #f5f5f5;
-  margin-left: 20px;
-  white-space: pre-wrap;
-}
-
-.instance-reference {
-  font-style: italic;
-  background-color: #f5f5f5;
-  white-space: pre-wrap;
-}
-
-.taxonomy-table {
-  margin: 10px;
-}
-
-thead .table-sort-column {
-  background-color: #ffe599;
-}
-
-tbody .table-sort-column {
-  background-color: #fff2cc;
-}
-
-.prompt {
-  font-style: italic;
-  background-color: #f5f5f5;
-  white-space: pre-wrap;
-}
-
-.plot {
-  margin: 15px;
-}
-
-.plot img {
-  margin: 10px;
-}
-
-.plot-caption {
-  color: #555;
-  font-style: italic;
-  margin: 5px;
-}
-
-.prediction-text {
-  white-space: pre-wrap;
-}