crfm-helm 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

helm/config/tokenizer_configs.yaml

@@ -69,7 +69,7 @@ tokenizer_configs:
     tokenizer_spec:
       class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
-    prefix_token: "</s>"
+    prefix_token: "<s>"
   - name: bigscience/T0pp
     tokenizer_spec:
       class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
@@ -83,6 +83,46 @@ tokenizer_configs:
     end_of_text_token: ""
     prefix_token: ":"
 
+  - name: cohere/command
+    tokenizer_spec:
+      class_name: "helm.tokenizers.cohere_tokenizer.CohereLocalTokenizer"
+    end_of_text_token: "<EOS_TOKEN>"
+    prefix_token: "<BOS_TOKEN>"
+
+  - name: cohere/command-light
+    tokenizer_spec:
+      class_name: "helm.tokenizers.cohere_tokenizer.CohereLocalTokenizer"
+    end_of_text_token: "<EOS_TOKEN>"
+    prefix_token: "<BOS_TOKEN>"
+
+  - name: cohere/command-r
+    tokenizer_spec:
+      class_name: "helm.tokenizers.cohere_tokenizer.CohereLocalTokenizer"
+    end_of_text_token: "<EOS_TOKEN>"
+    prefix_token: "<BOS_TOKEN>"
+
+  - name: cohere/command-r-plus
+    tokenizer_spec:
+      class_name: "helm.tokenizers.cohere_tokenizer.CohereLocalTokenizer"
+    end_of_text_token: "<EOS_TOKEN>"
+    prefix_token: "<BOS_TOKEN>"
+
+  - name: cohere/c4ai-command-r-v01
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: CohereForAI/c4ai-command-r-v01
+    end_of_text_token: "<EOS_TOKEN>"
+    prefix_token: "<BOS_TOKEN>"
+
+  - name: cohere/c4ai-command-r-plus
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: CohereForAI/c4ai-command-r-plus
+    end_of_text_token: "<EOS_TOKEN>"
+    prefix_token: "<BOS_TOKEN>"
+
   # Databricks
   - name: databricks/dbrx-instruct
     tokenizer_spec:
@@ -247,6 +287,17 @@ tokenizer_configs:
     end_of_text_token: "</s>"
     prefix_token: "<s>"
 
+  # AI Singapore
+  - name: aisingapore/sea-lion-7b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        trust_remote_code: true
+        use_fast: false
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
+
 
   # Allen Institute for AI
   # The allenai/olmo-7b requires Python 3.9 or newer.
@@ -259,6 +310,12 @@ tokenizer_configs:
     end_of_text_token: "<|endoftext|>"
     prefix_token: ""
 
+  - name: allenai/OLMo-1.7-7B-hf
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
 
   # Microsoft
   - name: microsoft/phi-2  
@@ -274,6 +331,24 @@ tokenizer_configs:
     end_of_text_token: "</s>"
     prefix_token: "<s>"
 
+  - name: mistralai/Mistral-7B-Instruct-v0.1
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+  - name: mistralai/Mistral-7B-Instruct-v0.2
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+  - name: mistralai/Mistral-7B-Instruct-v0.3
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
   # Neurips
   - name: neurips/local
     tokenizer_spec:
@@ -288,6 +363,12 @@ tokenizer_configs:
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
 
+  - name: openai/o200k_base
+    tokenizer_spec:
+      class_name: "helm.tokenizers.tiktoken_tokenizer.TiktokenTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
   - name: openai/clip-vit-large-patch14
     tokenizer_spec:
       class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
@@ -311,6 +392,14 @@ tokenizer_configs:
     end_of_text_token: "<|endoftext|>"
     prefix_token: ""
 
+  - name: qwen/qwen2-72b-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Qwen/Qwen2-72B-Instruct
+    end_of_text_token: <|im_end|>"
+    prefix_token: "<|im_start|>'"
+
   - name: qwen/qwen-vl
     tokenizer_spec:
       class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
@@ -331,6 +420,16 @@ tokenizer_configs:
     end_of_text_token: "<|endoftext|>"
     prefix_token: ""
 
+  # Snowflake
+  - name: snowflake/snowflake-arctic-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Snowflake/snowflake-arctic-instruct
+        trust_remote_code: true
+    end_of_text_token: "<|im_end|>"
+    prefix_token: "<|im_start|>"
+
   # Tiiuae
   - name: tiiuae/falcon-7b
     tokenizer_spec:

helm/proxy/critique/model_critique_client.py

@@ -15,6 +15,7 @@ from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import Request, RequestResult, GeneratedOutput
 from helm.clients.client import Client
 from helm.proxy.critique.critique_client import CritiqueClient
+from helm.common.media_object import MultimediaObject, MediaObject
 
 
 class CritiqueParseError(Exception):
@@ -24,6 +25,8 @@ class CritiqueParseError(Exception):
 class ModelCritiqueClient(CritiqueClient):
     """A CritiqueClient that queries a Model to answer CritiqueRequests."""
 
+    VISION_LANGUAGE_MODELS = ["openai/gpt-4-vision", "reka/reka", "huggingface/prometheus-vision"]
+
     def __init__(self, client: Client, model_name):
         self._client = client
         self._model_name = model_name
@@ -31,6 +34,11 @@ class ModelCritiqueClient(CritiqueClient):
             get_default_model_deployment_for_model(model_name, warn_arg_deprecated=False, ignore_deprecated=True)
             or self._model_name
         )
+        self.vision_language = False
+        for vision_language_model_name in self.VISION_LANGUAGE_MODELS:
+            if model_name.startswith(vision_language_model_name):
+                self.vision_language = True
+                break
 
     def _interpolate_fields(self, text: str, fields: Dict[str, str]) -> str:
         for key, value in fields.items():
@@ -58,10 +66,15 @@ class ModelCritiqueClient(CritiqueClient):
 
         requests: List[Request] = []
         for question in task.questions:
-            prompt: str = base_prompt + "\n\n" + self._question_to_prompt(question, fields)
+            prompt: str
+            if len(question.text) > 0:
+                prompt = base_prompt + "\n\n" + self._question_to_prompt(question, fields)
+            else:
+                # We may don't want to add extra newlines and prompts
+                # if the question text is empty (e.g., the Vibe-Eval evaluator).
+                prompt = base_prompt
             if question.question_type == "free_response":
-                # TODO: Make max_tokens configurable
-                max_tokens = 100
+                max_tokens = 100 if task.max_tokens is None else task.max_tokens
             elif question.question_type == "checkbox":
                 # We multiply by 2 because the model will generate a comma after each option.
                 max_tokens = len(question.options) * 2
@@ -78,12 +91,21 @@ class ModelCritiqueClient(CritiqueClient):
 
                 prompt = anthropic.HUMAN_PROMPT + prompt + anthropic.AI_PROMPT
 
+            multimodal_prompt: Optional[MultimediaObject] = None
+            if self.vision_language:
+                assert question.media_object is not None, "Expect media_object for vision-language models"
+                image_media: MediaObject = question.media_object
+                text_media: MediaObject = MediaObject(text=prompt, content_type="text/plain")
+                multimodal_prompt = MultimediaObject(media_objects=[image_media, text_media])
+                prompt = ""  # set to empty string to avoid conflicts with multimodal_prompt
+
             request = Request(
                 model=self._model_name,
                 model_deployment=self._model_deployment_name,
                 prompt=prompt,
                 max_tokens=max_tokens,
                 echo_prompt=False,
+                multimodal_prompt=multimodal_prompt,
             )
             requests.append(request)
         return requests
@@ -124,7 +146,13 @@ class ModelCritiqueClient(CritiqueClient):
                 raise CritiqueParseError(
                     f"Invalid answer: {completion}. Multiple choice questions should have one answer."
                 )
-            return answers[0]
+            letter_answer = answers[0]
+            choice_rank = string.ascii_uppercase.index(letter_answer)
+            if choice_rank >= len(question.options):
+                raise CritiqueParseError(
+                    f"Invalid answer: {completion}. The answer is out of range of the options: {question.options}"
+                )
+            return letter_answer
         except CritiqueParseError as e:
             # If there was an error parsing the answer, we assume the user did not answer the question.
             hlog(f"Error parsing answer: {e}. Skipping question (and so the respondent entirely)")

helm/proxy/services/server_service.py

@@ -119,7 +119,7 @@ class ServerService(Service):
                 return "codex"
             elif model_deployment.startswith("openai/dall-e-"):
                 return "dall_e"
-            elif model_deployment.startswith("openai/gpt-4-"):
+            elif model_deployment.startswith("openai/gpt-4"):
                 return "gpt4"
             else:
                 return "gpt3"

helm/tokenizers/auto_tokenizer.py

@@ -41,7 +41,7 @@ class AutoTokenizer(Tokenizer):
         if tokenizer_config:
             tokenizer_spec = inject_object_spec_args(
                 tokenizer_config.tokenizer_spec,
-                constant_bindings={"cache_config": cache_config},
+                constant_bindings={"cache_config": cache_config, "tokenizer_name": tokenizer_name},
                 provider_bindings={
                     "api_key": lambda: provide_api_key(self.credentials, organization),
                     "project_id": lambda: self.credentials.get(organization + "ProjectId", None),  # VertexAI

helm/tokenizers/cohere_tokenizer.py

@@ -1,6 +1,9 @@
 import json
 import requests
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
+
+import cohere
+from cohere.manually_maintained.tokenizers import get_hf_tokenizer
 
 from helm.common.cache import CacheConfig
 from helm.common.tokenization_request import (
@@ -10,7 +13,7 @@ from helm.common.tokenization_request import (
     TokenizationToken,
 )
 from helm.clients.cohere_utils import get_cohere_url, DEFAULT_COHERE_API_VERSION
-from .caching_tokenizer import CachingTokenizer
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
 
 
 class CohereTokenizer(CachingTokenizer):
@@ -81,3 +84,42 @@ class CohereTokenizer(CachingTokenizer):
 
     def decode(self, request: DecodeRequest) -> DecodeRequestResult:
         raise NotImplementedError("The Cohere API does not support decoding.")
+
+
+class CohereLocalTokenizer(CachingTokenizer):
+    """Cohere tokenizer using the Cohere Python library."""
+
+    def __init__(self, api_key: Optional[str], cache_config: CacheConfig) -> None:
+        super().__init__(cache_config)
+        self.client = cohere.Client(api_key)
+
+    def _tokenization_request_to_cache_key(self, request: TokenizationRequest) -> Dict[str, Any]:
+        return {"text": request.text, "tokenizer": request.tokenizer}
+
+    def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        model: str = request["tokenizer"].split("/")[1]
+        # Workaround for https://github.com/cohere-ai/cohere-python/issues/493
+        # `token_strings` are always set to `[]`, so we have to populate it ourselves.
+        response = self.client.tokenize(text=request["text"], model=model)
+        response_dict = response.dict()
+        response_dict["token_strings"] = get_hf_tokenizer(self.client, model).decode_batch(
+            [[token] for token in response.tokens]
+        )
+        return response_dict
+
+    def _tokenization_raw_response_to_tokens(
+        self, response: Dict[str, Any], request: TokenizationRequest
+    ) -> List[TokenizationToken]:
+        tokens: List[TokenizationToken] = []
+        if request.encode:
+            tokens = [TokenizationToken(token) for token in response["tokens"]]
+        else:
+            tokens = [TokenizationToken(token) for token in response["token_strings"]]
+        if request.truncation:
+            tokens = tokens[: request.max_length]
+        return tokens
+
+    def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        model: str = request["tokenizer"].split("/")[1]
+        response = self.client.detokenize(tokens=request["tokens"], model=model)
+        return response.dict()

helm/tokenizers/huggingface_tokenizer.py

@@ -29,8 +29,17 @@ class HuggingFaceTokenizer(CachingTokenizer):
     _tokenizers: Dict[str, WrappedPreTrainedTokenizer] = {}
     _tokenizers_lock: Lock = Lock()
 
-    def __init__(self, cache_config: CacheConfig, pretrained_model_name_or_path: Optional[str] = None, **kwargs):
+    def __init__(
+        self,
+        cache_config: CacheConfig,
+        tokenizer_name: str,
+        pretrained_model_name_or_path: Optional[str] = None,
+        **kwargs,
+    ):
         super().__init__(cache_config=cache_config)
+        self._helm_tokenizer_name = (
+            tokenizer_name  # HELM tokenizer name (e.g. "huggingface/gpt2"), *not* Hugging Face Hub Model ID
+        )
         self._pretrained_model_name_or_path = pretrained_model_name_or_path
         self._kwargs = kwargs
 
@@ -40,7 +49,11 @@ class HuggingFaceTokenizer(CachingTokenizer):
         # To avoid deadlocks when using HuggingFace tokenizers with multiple processes
         # TODO: Figure out if we actually need this.
         os.environ["TOKENIZERS_PARALLELISM"] = "False"
-
+        from_pretrained_kwargs = {**kwargs}
+        # If unspecified, set `use_fast=True` by default.
+        if "use_fast" not in from_pretrained_kwargs:
+            from_pretrained_kwargs["use_fast"] = True
+        print(from_pretrained_kwargs)
         try:
             # From the Hugging Face documentation, "local_files_only(defaults to False) —
             # Whether or not to only look at local files".
@@ -53,14 +66,14 @@ class HuggingFaceTokenizer(CachingTokenizer):
             # Tokenizers, which are written in Rust." So, use the "fast" version of the tokenizers if available.
             return WrappedPreTrainedTokenizer(
                 AutoTokenizer.from_pretrained(
-                    pretrained_model_name_or_path, local_files_only=True, use_fast=True, **kwargs
+                    pretrained_model_name_or_path, local_files_only=True, **from_pretrained_kwargs
                 )
             )
         except OSError:
             hlog(f"Local files do not exist for HuggingFace tokenizer: {pretrained_model_name_or_path}. Downloading...")
             return WrappedPreTrainedTokenizer(
                 AutoTokenizer.from_pretrained(
-                    pretrained_model_name_or_path, local_files_only=False, use_fast=True, **kwargs
+                    pretrained_model_name_or_path, local_files_only=False, **from_pretrained_kwargs
                 )
             )
 
@@ -84,21 +97,26 @@ class HuggingFaceTokenizer(CachingTokenizer):
                     )
         return HuggingFaceTokenizer._tokenizers[helm_tokenizer_name]
 
-    def _get_tokenizer_for_request(self, request: Dict[str, Any]) -> WrappedPreTrainedTokenizer:
-        """Method used in both _tokenize_do_it and _decode_do_it to get the tokenizer."""
+    def get_wrapped_tokenizer(self) -> WrappedPreTrainedTokenizer:
+        """Get the underlying Hugging Face WrappedPreTrainedTokenizer."""
         pretrained_model_name_or_path = (
-            self._pretrained_model_name_or_path if self._pretrained_model_name_or_path else request["tokenizer"]
+            self._pretrained_model_name_or_path if self._pretrained_model_name_or_path else self._helm_tokenizer_name
         )
         return HuggingFaceTokenizer.get_tokenizer(
-            helm_tokenizer_name=request["tokenizer"],
+            helm_tokenizer_name=self._helm_tokenizer_name,
             pretrained_model_name_or_path=pretrained_model_name_or_path,
             **self._kwargs,
         )
 
     def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        if request["tokenizer"] != self._helm_tokenizer_name:
+            raise ValueError(
+                f"This HuggingFaceTokenizer expects tokenizer to be {self._helm_tokenizer_name} "
+                "but instead the request has tokenizer {request['tokenizer']}"
+            )
         if request["encode"]:
             if request["truncation"]:
-                with self._get_tokenizer_for_request(request) as tokenizer:
+                with self.get_wrapped_tokenizer() as tokenizer:
                     tokens = tokenizer.encode(
                         request["text"],
                         truncation=request["truncation"],
@@ -106,7 +124,7 @@ class HuggingFaceTokenizer(CachingTokenizer):
                         add_special_tokens=False,
                     )
             else:
-                with self._get_tokenizer_for_request(request) as tokenizer:
+                with self.get_wrapped_tokenizer() as tokenizer:
                     tokens = tokenizer.encode(request["text"], add_special_tokens=False)
         else:
             if "gpt" in request["tokenizer"] or request["tokenizer"] in [
@@ -118,7 +136,7 @@ class HuggingFaceTokenizer(CachingTokenizer):
                 # convert_tokens_to_string method. We prefer to use this method instead
                 # of the hacky cleanup_tokens method below as it might handle cases
                 # we haven't thought of in cleanup_tokens.
-                with self._get_tokenizer_for_request(request) as tokenizer:
+                with self.get_wrapped_tokenizer() as tokenizer:
                     tokens = [
                         tokenizer.convert_tokens_to_string([token]) for token in tokenizer.tokenize(request["text"])
                     ]
@@ -131,7 +149,7 @@ class HuggingFaceTokenizer(CachingTokenizer):
                 # But this replaces all the "▁" characters by "", which is not what we want.
                 # This would be problematic as tokenize(" Hello", encode=False) would return ["Hello"]
                 # Just like tokenize("Hello", encode=False) would return ["Hello"].
-                with self._get_tokenizer_for_request(request) as tokenizer:
+                with self.get_wrapped_tokenizer() as tokenizer:
                     tokens = tokenizer.tokenize(request["text"])
                 # Some tokenizers (e.g. Qwen/Qwen-7B) return the tokens as bytes, so we have to decode them to strings.
                 if tokens and type(tokens[0]) == bytes:
@@ -140,7 +158,12 @@ class HuggingFaceTokenizer(CachingTokenizer):
         return {"tokens": tokens}
 
     def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
-        with self._get_tokenizer_for_request(request) as tokenizer:
+        if request["tokenizer"] != self._helm_tokenizer_name:
+            raise ValueError(
+                f"This HuggingFaceTokenizer expects tokenizer to be {self._helm_tokenizer_name} "
+                "but instead the request has tokenizer {request['tokenizer']}"
+            )
+        with self.get_wrapped_tokenizer() as tokenizer:
             text = tokenizer.decode(
                 request["tokens"], clean_up_tokenization_spaces=request["clean_up_tokenization_spaces"]
             )

helm/tokenizers/test_cohere_tokenizer.py

@@ -0,0 +1,39 @@
+import pytest
+
+from helm.common.cache import BlackHoleCacheConfig
+from helm.common.tokenization_request import (
+    DecodeRequest,
+    TokenizationRequest,
+    TokenizationToken,
+)
+from helm.tokenizers.cohere_tokenizer import CohereLocalTokenizer
+
+
+@pytest.mark.models
+def test_tokenize():
+    tokenizer = CohereLocalTokenizer(api_key=None, cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="cohere/command", text="otter 🦦")
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in ["ot", "ter", " �", "�", "�"]]
+
+
+@pytest.mark.models
+def test_encode():
+    tokenizer = CohereLocalTokenizer(api_key=None, cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="cohere/command", text="otter 🦦", encode=True)
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in [1741, 1779, 7728, 107, 107]]
+
+
+@pytest.mark.models
+def test_decode():
+    tokenizer = CohereLocalTokenizer(api_key=None, cache_config=BlackHoleCacheConfig())
+    request = DecodeRequest(tokenizer="cohere/command", tokens=[1741, 1779, 7728, 107, 107])
+    result = tokenizer.decode(request)
+    assert result.success
+    assert not result.cached
+    assert result.text == "otter 🦦"

helm/tokenizers/test_huggingface_tokenizer.py

@@ -17,7 +17,11 @@ class TestHuggingFaceGPT2Tokenizer:
     def setup_method(self, method):
         cache_file = tempfile.NamedTemporaryFile(delete=False)
         self.cache_path: str = cache_file.name
-        self.tokenizer = HuggingFaceTokenizer(SqliteCacheConfig(self.cache_path))
+        self.tokenizer = HuggingFaceTokenizer(
+            SqliteCacheConfig(self.cache_path),
+            tokenizer_name="huggingface/gpt2",
+            pretrained_model_name_or_path="openai-community/gpt2",
+        )
 
     def teardown_method(self, method):
         os.remove(self.cache_path)