crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/config/tokenizer_configs.yaml

@@ -0,0 +1,362 @@
+# This file defines all the tokenizers that are supported by the Helm API.
+
+# If you want to add a new tokenizer, you can technically do it here but we recommend
+# you to do it in prod_env/tokenizer_configs.yaml instead.
+
+# Follow the template of this file to add a new tokenizer. You can copy paste this to get started:
+#    # This file contains the tokenizer configs for the private tokenizers
+#    tokenizer_configs: [] # Leave empty to disable private tokenizers
+
+
+tokenizer_configs:
+
+  - name: simple/tokenizer1
+    tokenizer_spec:
+      class_name: "helm.tokenizers.simple_tokenizer.SimpleTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+  # AI21
+  - name: ai21/j1
+    tokenizer_spec:
+      class_name: "helm.tokenizers.ai21_tokenizer.AI21Tokenizer"
+    end_of_text_token: " "
+    prefix_token: ""
+
+  # AlephAlpha
+  - name: AlephAlpha/luminous-base
+    tokenizer_spec:
+      class_name: "helm.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"
+    end_of_text_token: ""
+    prefix_token: ""
+  - name: AlephAlpha/luminous-extended
+    tokenizer_spec:
+      class_name: "helm.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"
+    end_of_text_token: ""
+    prefix_token: ""
+  - name: AlephAlpha/luminous-supreme
+    tokenizer_spec:
+      class_name: "helm.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"
+    end_of_text_token: ""
+    prefix_token: ""
+  - name: AlephAlpha/luminous-world
+    tokenizer_spec:
+      class_name: "helm.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"
+    end_of_text_token: ""
+    prefix_token: ""
+
+  # Anthropic
+  - name: anthropic/claude
+    tokenizer_spec:
+      class_name: "helm.tokenizers.anthropic_tokenizer.AnthropicTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  # Bigcode
+  - name: bigcode/santacoder
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+  - name: bigcode/starcoder
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  # Bigscience
+  - name: bigscience/bloom
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "</s>"
+  - name: bigscience/T0pp
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: ""
+
+  # Cohere
+  - name: cohere/cohere
+    tokenizer_spec:
+      class_name: "helm.tokenizers.cohere_tokenizer.CohereTokenizer"
+    end_of_text_token: ""
+    prefix_token: ":"
+
+  # Databricks
+  - name: databricks/dbrx-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  # DeepSeek
+  - name: deepseek-ai/deepseek-llm-67b-chat
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<｜end▁of▁sentence｜>"
+    prefix_token: "<｜begin▁of▁sentence｜>"
+
+  # EleutherAI
+  - name: EleutherAI/gpt-j-6B
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+  - name: EleutherAI/gpt-neox-20b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  # Facebook
+  - name: facebook/opt-66b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "</s>"
+
+  # Google
+  - name: google/t5-11b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: google-t5/t5-11b
+    end_of_text_token: "</s>"
+    prefix_token: ""
+  - name: google/flan-t5-xxl
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: ""
+  - name: google/ul2
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: ""
+  - name: google/mt5-base
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: ""
+  - name: google/text-bison@001
+    tokenizer_spec:
+      class_name: "helm.tokenizers.vertexai_tokenizer.VertexAITokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: ""
+  - name: google/text-bison@002
+    tokenizer_spec:
+      class_name: "helm.tokenizers.vertexai_tokenizer.VertexAITokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: ""
+  - name: google/text-unicorn@001
+    tokenizer_spec:
+      class_name: "helm.tokenizers.vertexai_tokenizer.VertexAITokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: ""
+  - name: google/gemma-2b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<eos>"
+    prefix_token: "<bos>"
+
+  # Hf-internal-testing
+
+  # Tokenizer name hf-internal-testing/llama-tokenizer is taken from:
+  # https://huggingface.co/docs/transformers/main/en/model_doc/llama#transformers.LlamaTokenizerFast.example
+  - name: hf-internal-testing/llama-tokenizer
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+  # HuggingFaceM4
+  - name: HuggingFaceM4/idefics-9b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+  - name: HuggingFaceM4/idefics-9b-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+  - name: HuggingFaceM4/idefics-80b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+  - name: HuggingFaceM4/idefics-80b-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+    
+  - name: anas-awadalla/mpt-7b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
+  # Huggingface
+  - name: huggingface/gpt2
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: openai-community/gpt2
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  # Lighting AI
+  - name: lightningai/lit-gpt
+    tokenizer_spec:
+      class_name: "helm.tokenizers.lit_gpt_tokenizer.LitGPTTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  # Meta-llama
+
+  # To use the Llama-2 tokenizer:
+    #
+    # 1. Accept the license agreement: https://ai.meta.com/resources/models-and-libraries/llama-downloads/
+    # 2. Request to access the Hugging Face repository: https://huggingface.co/meta-llama/Llama-2-7b
+    # 3. Run `huggingface-cli login`
+    #
+    # If you encounter the following error, complete the above steps and try again:
+    #
+    #     meta-llama/Llama-2-70b-hf is not a local folder and is not a valid model identifier listed on
+    #     'https://huggingface.co/models'
+  - name: meta-llama/Llama-2-7b-hf
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+  - name: meta/llama-3-8b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: meta-llama/Meta-Llama-3-8B
+    prefix_token: "<|begin_of_text|>"
+    end_of_text_token: "<|end_of_text|>"
+
+  # 01-ai
+  - name: 01-ai/Yi-6B
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+
+  # Allen Institute for AI
+  # The allenai/olmo-7b requires Python 3.9 or newer.
+  # To use the allenai/olmo-7b tokenizer, run `pip install crfm-helm[allenai]` first.
+  - name: allenai/olmo-7b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        trust_remote_code: true
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
+
+  # Microsoft
+  - name: microsoft/phi-2  
+    tokenizer_spec:  
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"  
+    end_of_text_token: "<|endoftext|>"  
+    prefix_token: "<|endoftext|>"
+
+  # Mistralai
+  - name: mistralai/Mistral-7B-v0.1
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+  # Neurips
+  - name: neurips/local
+    tokenizer_spec:
+      class_name: "helm.tokenizers.http_model_tokenizer.HTTPModelTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  # Openai
+  - name: openai/cl100k_base
+    tokenizer_spec:
+      class_name: "helm.tokenizers.tiktoken_tokenizer.TiktokenTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  - name: openai/clip-vit-large-patch14
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: ""
+    prefix_token: ""
+
+  - name: qwen/qwen-7b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Qwen/Qwen-7B
+        trust_remote_code: true
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
+  - name: qwen/qwen1.5-7b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Qwen/Qwen1.5-7B
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
+  - name: qwen/qwen-vl
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Qwen/Qwen-VL
+        trust_remote_code: true
+    # Source: https://github.com/QwenLM/Qwen-VL
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
+  - name: qwen/qwen-vl-chat
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Qwen/Qwen-VL-Chat
+        trust_remote_code: true
+    # Source: https://github.com/QwenLM/Qwen-VL
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
+  # Tiiuae
+  - name: tiiuae/falcon-7b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
+  # TsinghuaKEG
+  - name: TsinghuaKEG/ice
+    tokenizer_spec:
+      class_name: "helm.tokenizers.ice_tokenizer.ICETokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: ""
+
+  # Writer
+  - name: writer/gpt2
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: openai-community/gpt2
+    end_of_text_token: ""
+    prefix_token: ""
+
+  # Yandex
+  - name: Yandex/yalm
+    tokenizer_spec:
+      class_name: "helm.tokenizers.yalm_tokenizer.YaLMTokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: "</s>"

helm/proxy/accounts.py

@@ -23,6 +23,9 @@ DEFAULT_QUOTAS = {
     "jurassic": {"daily": 10000},
     "gooseai": {"daily": 10000},
     "cohere": {"daily": 10000},
+    "dall_e": {"daily": 5},  # In terms of the number of generated images
+    "together_vision": {"daily": 30},
+    "simple": {"daily": 10000},
 }
 
 
@@ -303,7 +306,7 @@ class Accounts:
             model_group: str,
             granularity: str,
             compute_period: Callable[[], str],
-        ):
+        ) -> None:
             """Helper that checks the usage at a certain granularity (e.g., daily, monthly, total)."""
 
             model_group_usages = account.usages.get(model_group)
@@ -321,14 +324,38 @@ class Accounts:
             if not usage.can_use():
                 raise InsufficientQuotaError(f"{granularity} quota ({usage.quota}) for {model_group} already used up")
 
+        def check_non_empty_quota(
+            account: Account,
+            model_group: str,
+        ) -> None:
+            """Helper that checks that the account has quota at some granularity.
+
+            At each granularity, a quota of None means unlimited quota.
+            However, if the quota is None at every granularity, it means that there is no quota.
+            To enforce this rule, this helper raises a InsufficientQuotaError if the quota is None
+            at every granularity."""
+            model_group_usages = account.usages.get(model_group)
+            if model_group_usages is None:
+                raise InsufficientQuotaError(f"No quota for {model_group}")
+            if all(
+                [
+                    granularity_usage.quota is None or granularity_usage.quota <= 0
+                    for granularity_usage in model_group_usages.values()
+                ]
+            ):
+                raise InsufficientQuotaError(f"No quota for {model_group}")
+
         if self.root_mode:
             return
 
         with SqliteDict(self.path) as cache:
             account: Account = from_dict(Account, cache[api_key])
-            granular_check_can_use(account, model_group, "daily", compute_daily_period)
-            granular_check_can_use(account, model_group, "monthly", compute_monthly_period)
-            granular_check_can_use(account, model_group, "total", compute_total_period)
+        if account.is_admin:
+            return
+        granular_check_can_use(account, model_group, "daily", compute_daily_period)
+        granular_check_can_use(account, model_group, "monthly", compute_monthly_period)
+        granular_check_can_use(account, model_group, "total", compute_total_period)
+        check_non_empty_quota(account, model_group)
 
     def use(self, api_key: str, model_group: str, delta: int):
         """

helm/proxy/critique/mechanical_turk_critique_importer.py

@@ -4,6 +4,7 @@ import os
 from threading import Lock
 from typing import Dict, List, Optional, Tuple, Union
 import re
+import sys
 
 from helm.common.critique_request import (
     CritiqueRequest,
@@ -15,6 +16,8 @@ from helm.common.critique_request import (
 from helm.common.hierarchical_logger import hlog
 from helm.proxy.critique.mechanical_turk_utils import replace_emoji_characters
 
+csv.field_size_limit(sys.maxsize)
+
 # A representation of fields that can be used as a dict key.
 _CritiqueRequestKey = Tuple[Tuple[str, str], ...]
 

helm/proxy/critique/model_critique_client.py

@@ -2,6 +2,7 @@ from typing import Dict, List, Union, Optional
 import string
 import dataclasses
 
+from helm.benchmark.run_spec_factory import get_default_model_deployment_for_model
 from helm.common.critique_request import (
     CritiqueRequest,
     CritiqueRequestResult,
@@ -11,8 +12,8 @@ from helm.common.critique_request import (
 )
 from helm.common.hierarchical_logger import hlog
 from helm.common.optional_dependencies import handle_module_not_found_error
-from helm.common.request import Request, RequestResult, Sequence
-from helm.proxy.clients.client import Client
+from helm.common.request import Request, RequestResult, GeneratedOutput
+from helm.clients.client import Client
 from helm.proxy.critique.critique_client import CritiqueClient
 
 
@@ -26,6 +27,10 @@ class ModelCritiqueClient(CritiqueClient):
     def __init__(self, client: Client, model_name):
         self._client = client
         self._model_name = model_name
+        self._model_deployment_name = (
+            get_default_model_deployment_for_model(model_name, warn_arg_deprecated=False, ignore_deprecated=True)
+            or self._model_name
+        )
 
     def _interpolate_fields(self, text: str, fields: Dict[str, str]) -> str:
         for key, value in fields.items():
@@ -75,6 +80,7 @@ class ModelCritiqueClient(CritiqueClient):
 
             request = Request(
                 model=self._model_name,
+                model_deployment=self._model_deployment_name,
                 prompt=prompt,
                 max_tokens=max_tokens,
                 echo_prompt=False,
@@ -108,7 +114,7 @@ class ModelCritiqueClient(CritiqueClient):
         return answers
 
     def _multiple_choice_completion_to_answer(
-        self, question: CritiqueQuestionTemplate, completion: Sequence
+        self, question: CritiqueQuestionTemplate, completion: GeneratedOutput
     ) -> Optional[str]:
         """Convert a multiple choice completion to an answer."""
         assert question.question_type == "multiple_choice"
@@ -125,7 +131,7 @@ class ModelCritiqueClient(CritiqueClient):
             return None
 
     def _checkbox_completion_to_answer(
-        self, question: CritiqueQuestionTemplate, completion: Sequence
+        self, question: CritiqueQuestionTemplate, completion: GeneratedOutput
     ) -> Optional[List[str]]:
         """Convert a checkbox completion to an answer."""
         assert question.question_type == "checkbox"
@@ -141,7 +147,9 @@ class ModelCritiqueClient(CritiqueClient):
             hlog(f"Error parsing answer: {e}. Skipping question (and so the respondent entirely)")
             return None
 
-    def _free_response_completion_to_answer(self, question: CritiqueQuestionTemplate, completion: Sequence) -> str:
+    def _free_response_completion_to_answer(
+        self, question: CritiqueQuestionTemplate, completion: GeneratedOutput
+    ) -> str:
         """Convert a free response completion to an answer."""
         assert question.question_type == "free_response"
         return completion.text

helm/proxy/example_queries.py

@@ -21,6 +21,8 @@ example_queries = [
             """
             temperature: 0.5  # Medium amount of randomness
             stop_sequences: [.]  # Stop when you hit a period
+            model: openai/gpt-3.5-turbo-0613
+            model_deployment: openai/gpt-3.5-turbo-0613
             """
         ),
         environments="",
@@ -31,7 +33,9 @@ example_queries = [
             """
             temperature: 0.5  # Medium amount of randomness
             stop_sequences: [\\n]  # Stop when you hit a newline
-            num_completions: 10  # Generate many samples
+            num_completions: 5  # Generate many samples
+            model: openai/gpt-3.5-turbo-0613
+            model_deployment: openai/gpt-3.5-turbo-0613
             """
         ),
         environments="",
@@ -42,7 +46,9 @@ example_queries = [
             """
             echo_prompt: true  # Analyze the prompt
             max_tokens: 0  # Don't generate any more
-            top_k_per_token: 10  # Show alternatives for each position
+            top_k_per_token: 5  # Show alternatives for each position
+            model: openai/davinci-002
+            model_deployment: openai/davinci-002
             """
         ),
         environments=dedent(""),
@@ -53,6 +59,8 @@ example_queries = [
             """
             temperature: 0  # Deterministic
             max_tokens: 50
+            model: openai/gpt-3.5-turbo-0613
+            model_deployment: openai/gpt-3.5-turbo-0613
             """
         ),
         environments="",
@@ -63,13 +71,15 @@ example_queries = [
             """
             temperature: 0
             stop_sequences: [.]
-            model: ${model}  # Try out multiple models
+            # Try out multiple models
+            model: ${model}
+            model_deployment: ${model}
             """
         ),
         environments=dedent(
             """
             occupation: [mathematician, lawyer, doctor]
-            model: [openai/davinci, ai21/j1-jumbo]
+            model: [openai/gpt-3.5-turbo-0613, openai/gpt-3.5-turbo-1106]
             """
         ),
     ),
@@ -88,12 +98,14 @@ example_queries = [
             temperature: 0.5
             stop_sequences: [\\n]
             num_completions: 5
-            model: ${model}  # Try out GPT-3 and Jurassic
+            # Try out multiple models
+            model: ${model}
+            model_deployment: ${model}
             """
         ),
         environments=dedent(
             """
-            model: [openai/davinci, ai21/j1-jumbo]
+            model: [openai/gpt-3.5-turbo-0613, openai/gpt-3.5-turbo-1106]
             """
         ),
     ),
@@ -122,20 +134,23 @@ example_queries = [
             temperature: 0
             max_tokens: 1
             top_k_per_token: 4
-            model: ${model}  # Try out GPT-3 and Jurassic
+            # Try out multiple models
+            model: ${model}
+            model_deployment: ${model}
             """
         ),
         environments=dedent(
             """
-            model: [openai/davinci, ai21/j1-jumbo]
+            model: [openai/gpt-3.5-turbo-0613, openai/gpt-3.5-turbo-1106]
             """
         ),
     ),
     Query(
-        prompt="Takes two vectors a and b and returns their Euclidean distance",
+        prompt="Write a Python function that takes two vectors a and b and returns their Euclidean distance.",
         settings=dedent(
             """
-            model: openai/code-davinci-001  # Codex for code generation
+            model: openai/gpt-3.5-turbo-0613
+            model_deployment: openai/gpt-3.5-turbo-0613
             """
         ),
         environments="",
@@ -144,19 +159,16 @@ example_queries = [
         prompt="The quick brown fox",
         settings=dedent(
             """
-            model: ${model}
             temperature: 0.3
             stop_sequences: [\\n]
+            # Try out multiple models
+            model: ${model}
+            model_deployment: ${model}
             """
         ),
         environments=dedent(
             """
-            model: [
-                "openai/davinci", "openai/text-davinci-002",
-                "openai/text-davinci-003", "ai21/j1-grande-v2-beta",
-                "together/gpt-j-6b", "together/gpt-jt-6b-v1",
-                "together/bloom", "together/opt-175b"
-            ]
+            model: [openai/gpt-3.5-turbo-0613, openai/gpt-3.5-turbo-1106]
             """
         ),
     ),

helm/proxy/retry.py

@@ -41,11 +41,13 @@ def get_retry_decorator(
         Wait function to pass into `Retrying` that logs and returns the amount of time to sleep
         depending on the number of attempts and delay (in milliseconds).
         """
+        del delay  # unused
+        next_delay = 2**attempts * wait_exponential_multiplier_seconds * 1000
         hlog(
-            f"{operation} failed. Retrying (attempt #{attempts + 1}) in {delay // 1000} seconds... "
+            f"{operation} failed. Retrying (attempt #{attempts + 1}) in {next_delay // 1000} seconds... "
             "(See above for error details)"
         )
-        return _retrying.exponential_sleep(attempts, delay)
+        return next_delay
 
     def print_exception_and_traceback(exception: Exception) -> bool:
         """
@@ -85,3 +87,7 @@ def retry_if_request_failed(result: Union[RequestResult, TokenizationRequestResu
 retry_request: Callable = get_retry_decorator(
     "Request", max_attempts=5, wait_exponential_multiplier_seconds=5, retry_on_result=retry_if_request_failed
 )
+
+retry_tokenizer_request: Callable = get_retry_decorator(
+    "Request", max_attempts=5, wait_exponential_multiplier_seconds=1, retry_on_result=retry_if_request_failed
+)