crfm-helm 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/config/tokenizer_configs.yaml

@@ -10,129 +10,155 @@
 
 tokenizer_configs:
 
-  - name: simple/model1
+  - name: simple/tokenizer1
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.simple_tokenizer.SimpleTokenizer"
+      class_name: "helm.tokenizers.simple_tokenizer.SimpleTokenizer"
     end_of_text_token: "</s>"
     prefix_token: "<s>"
 
   # AI21
   - name: ai21/j1
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.ai21_tokenizer.AI21Tokenizer"
+      class_name: "helm.tokenizers.ai21_tokenizer.AI21Tokenizer"
     end_of_text_token: " "
     prefix_token: ""
 
   # AlephAlpha
   - name: AlephAlpha/luminous-base
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"
+      class_name: "helm.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"
     end_of_text_token: ""
     prefix_token: ""
   - name: AlephAlpha/luminous-extended
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"
+      class_name: "helm.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"
     end_of_text_token: ""
     prefix_token: ""
   - name: AlephAlpha/luminous-supreme
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"
+      class_name: "helm.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"
     end_of_text_token: ""
     prefix_token: ""
   - name: AlephAlpha/luminous-world
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"
+      class_name: "helm.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"
     end_of_text_token: ""
     prefix_token: ""
 
   # Anthropic
   - name: anthropic/claude
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.anthropic_tokenizer.AnthropicTokenizer"
+      class_name: "helm.tokenizers.anthropic_tokenizer.AnthropicTokenizer"
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
 
   # Bigcode
   - name: bigcode/santacoder
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
   - name: bigcode/starcoder
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
 
   # Bigscience
   - name: bigscience/bloom
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: "</s>"
   - name: bigscience/T0pp
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: ""
 
   # Cohere
   - name: cohere/cohere
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.cohere_tokenizer.CohereTokenizer"
+      class_name: "helm.tokenizers.cohere_tokenizer.CohereTokenizer"
     end_of_text_token: ""
     prefix_token: ":"
 
+  # Databricks
+  - name: databricks/dbrx-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  # DeepSeek
+  - name: deepseek-ai/deepseek-llm-67b-chat
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<｜end▁of▁sentence｜>"
+    prefix_token: "<｜begin▁of▁sentence｜>"
+
   # EleutherAI
   - name: EleutherAI/gpt-j-6B
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
   - name: EleutherAI/gpt-neox-20b
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
 
   # Facebook
   - name: facebook/opt-66b
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: "</s>"
 
   # Google
   - name: google/t5-11b
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: google-t5/t5-11b
     end_of_text_token: "</s>"
     prefix_token: ""
   - name: google/flan-t5-xxl
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: ""
   - name: google/ul2
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: ""
   - name: google/mt5-base
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: ""
   - name: google/text-bison@001
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.vertexai_tokenizer.VertexAITokenizer"
+      class_name: "helm.tokenizers.vertexai_tokenizer.VertexAITokenizer"
+    end_of_text_token: "</s>"
+    prefix_token: ""
+  - name: google/text-bison@002
+    tokenizer_spec:
+      class_name: "helm.tokenizers.vertexai_tokenizer.VertexAITokenizer"
     end_of_text_token: "</s>"
     prefix_token: ""
   - name: google/text-unicorn@001
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.vertexai_tokenizer.VertexAITokenizer"
+      class_name: "helm.tokenizers.vertexai_tokenizer.VertexAITokenizer"
     end_of_text_token: "</s>"
     prefix_token: ""
+  - name: google/gemma-2b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<eos>"
+    prefix_token: "<bos>"
 
   # Hf-internal-testing
 
@@ -140,43 +166,51 @@ tokenizer_configs:
   # https://huggingface.co/docs/transformers/main/en/model_doc/llama#transformers.LlamaTokenizerFast.example
   - name: hf-internal-testing/llama-tokenizer
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: "<s>"
 
   # HuggingFaceM4
   - name: HuggingFaceM4/idefics-9b
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: "<s>"
   - name: HuggingFaceM4/idefics-9b-instruct
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: "<s>"
   - name: HuggingFaceM4/idefics-80b
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: "<s>"
   - name: HuggingFaceM4/idefics-80b-instruct
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: "<s>"
+    
+  - name: anas-awadalla/mpt-7b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
 
   # Huggingface
   - name: huggingface/gpt2
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: openai-community/gpt2
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
 
   # Lighting AI
   - name: lightningai/lit-gpt
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.lit_gpt_tokenizer.LitGPTTokenizer"
+      class_name: "helm.tokenizers.lit_gpt_tokenizer.LitGPTTokenizer"
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
 
@@ -194,70 +228,135 @@ tokenizer_configs:
     #     'https://huggingface.co/models'
   - name: meta-llama/Llama-2-7b-hf
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: "<s>"
 
+  - name: meta/llama-3-8b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: meta-llama/Meta-Llama-3-8B
+    prefix_token: "<|begin_of_text|>"
+    end_of_text_token: "<|end_of_text|>"
 
   # 01-ai
   - name: 01-ai/Yi-6B
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: "<s>"
 
-  # Microsoft
-  - name: microsoft/gpt2
-    tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+
+  # Allen Institute for AI
+  # The allenai/olmo-7b requires Python 3.9 or newer.
+  # To use the allenai/olmo-7b tokenizer, run `pip install crfm-helm[allenai]` first.
+  - name: allenai/olmo-7b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        trust_remote_code: true
     end_of_text_token: "<|endoftext|>"
-    prefix_token: "<<"
+    prefix_token: ""
+
+
+  # Microsoft
+  - name: microsoft/phi-2  
+    tokenizer_spec:  
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"  
+    end_of_text_token: "<|endoftext|>"  
+    prefix_token: "<|endoftext|>"
 
   # Mistralai
   - name: mistralai/Mistral-7B-v0.1
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "</s>"
     prefix_token: "<s>"
 
   # Neurips
   - name: neurips/local
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.http_model_tokenizer.HTTPModelTokenizer"
+      class_name: "helm.tokenizers.http_model_tokenizer.HTTPModelTokenizer"
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
 
   # Openai
   - name: openai/cl100k_base
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.tiktoken_tokenizer.TiktokenTokenizer"
+      class_name: "helm.tokenizers.tiktoken_tokenizer.TiktokenTokenizer"
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
 
+  - name: openai/clip-vit-large-patch14
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: ""
+    prefix_token: ""
+
+  - name: qwen/qwen-7b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Qwen/Qwen-7B
+        trust_remote_code: true
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
+  - name: qwen/qwen1.5-7b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Qwen/Qwen1.5-7B
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
+  - name: qwen/qwen-vl
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Qwen/Qwen-VL
+        trust_remote_code: true
+    # Source: https://github.com/QwenLM/Qwen-VL
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
+  - name: qwen/qwen-vl-chat
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Qwen/Qwen-VL-Chat
+        trust_remote_code: true
+    # Source: https://github.com/QwenLM/Qwen-VL
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
   # Tiiuae
   - name: tiiuae/falcon-7b
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
     end_of_text_token: "<|endoftext|>"
     prefix_token: ""
 
   # TsinghuaKEG
   - name: TsinghuaKEG/ice
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.ice_tokenizer.ICETokenizer"
+      class_name: "helm.tokenizers.ice_tokenizer.ICETokenizer"
     end_of_text_token: "</s>"
     prefix_token: ""
 
   # Writer
   - name: writer/gpt2
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: openai-community/gpt2
     end_of_text_token: ""
     prefix_token: ""
 
   # Yandex
   - name: Yandex/yalm
     tokenizer_spec:
-      class_name: "helm.proxy.tokenizers.yalm_tokenizer.YaLMTokenizer"
+      class_name: "helm.tokenizers.yalm_tokenizer.YaLMTokenizer"
     end_of_text_token: "</s>"
     prefix_token: "</s>"

helm/proxy/accounts.py

@@ -23,6 +23,9 @@ DEFAULT_QUOTAS = {
     "jurassic": {"daily": 10000},
     "gooseai": {"daily": 10000},
     "cohere": {"daily": 10000},
+    "dall_e": {"daily": 5},  # In terms of the number of generated images
+    "together_vision": {"daily": 30},
+    "simple": {"daily": 10000},
 }
 
 
@@ -303,7 +306,7 @@ class Accounts:
             model_group: str,
             granularity: str,
             compute_period: Callable[[], str],
-        ):
+        ) -> None:
             """Helper that checks the usage at a certain granularity (e.g., daily, monthly, total)."""
 
             model_group_usages = account.usages.get(model_group)
@@ -321,14 +324,38 @@ class Accounts:
             if not usage.can_use():
                 raise InsufficientQuotaError(f"{granularity} quota ({usage.quota}) for {model_group} already used up")
 
+        def check_non_empty_quota(
+            account: Account,
+            model_group: str,
+        ) -> None:
+            """Helper that checks that the account has quota at some granularity.
+
+            At each granularity, a quota of None means unlimited quota.
+            However, if the quota is None at every granularity, it means that there is no quota.
+            To enforce this rule, this helper raises a InsufficientQuotaError if the quota is None
+            at every granularity."""
+            model_group_usages = account.usages.get(model_group)
+            if model_group_usages is None:
+                raise InsufficientQuotaError(f"No quota for {model_group}")
+            if all(
+                [
+                    granularity_usage.quota is None or granularity_usage.quota <= 0
+                    for granularity_usage in model_group_usages.values()
+                ]
+            ):
+                raise InsufficientQuotaError(f"No quota for {model_group}")
+
         if self.root_mode:
             return
 
         with SqliteDict(self.path) as cache:
             account: Account = from_dict(Account, cache[api_key])
-            granular_check_can_use(account, model_group, "daily", compute_daily_period)
-            granular_check_can_use(account, model_group, "monthly", compute_monthly_period)
-            granular_check_can_use(account, model_group, "total", compute_total_period)
+        if account.is_admin:
+            return
+        granular_check_can_use(account, model_group, "daily", compute_daily_period)
+        granular_check_can_use(account, model_group, "monthly", compute_monthly_period)
+        granular_check_can_use(account, model_group, "total", compute_total_period)
+        check_non_empty_quota(account, model_group)
 
     def use(self, api_key: str, model_group: str, delta: int):
         """

helm/proxy/critique/mechanical_turk_critique_importer.py

@@ -4,6 +4,7 @@ import os
 from threading import Lock
 from typing import Dict, List, Optional, Tuple, Union
 import re
+import sys
 
 from helm.common.critique_request import (
     CritiqueRequest,
@@ -15,6 +16,8 @@ from helm.common.critique_request import (
 from helm.common.hierarchical_logger import hlog
 from helm.proxy.critique.mechanical_turk_utils import replace_emoji_characters
 
+csv.field_size_limit(sys.maxsize)
+
 # A representation of fields that can be used as a dict key.
 _CritiqueRequestKey = Tuple[Tuple[str, str], ...]
 

helm/proxy/critique/model_critique_client.py

@@ -2,7 +2,7 @@ from typing import Dict, List, Union, Optional
 import string
 import dataclasses
 
-from helm.benchmark.run_specs import get_default_model_deployment_for_model
+from helm.benchmark.run_spec_factory import get_default_model_deployment_for_model
 from helm.common.critique_request import (
     CritiqueRequest,
     CritiqueRequestResult,
@@ -12,8 +12,8 @@ from helm.common.critique_request import (
 )
 from helm.common.hierarchical_logger import hlog
 from helm.common.optional_dependencies import handle_module_not_found_error
-from helm.common.request import Request, RequestResult, Sequence
-from helm.proxy.clients.client import Client
+from helm.common.request import Request, RequestResult, GeneratedOutput
+from helm.clients.client import Client
 from helm.proxy.critique.critique_client import CritiqueClient
 
 
@@ -114,7 +114,7 @@ class ModelCritiqueClient(CritiqueClient):
         return answers
 
     def _multiple_choice_completion_to_answer(
-        self, question: CritiqueQuestionTemplate, completion: Sequence
+        self, question: CritiqueQuestionTemplate, completion: GeneratedOutput
     ) -> Optional[str]:
         """Convert a multiple choice completion to an answer."""
         assert question.question_type == "multiple_choice"
@@ -131,7 +131,7 @@ class ModelCritiqueClient(CritiqueClient):
             return None
 
     def _checkbox_completion_to_answer(
-        self, question: CritiqueQuestionTemplate, completion: Sequence
+        self, question: CritiqueQuestionTemplate, completion: GeneratedOutput
     ) -> Optional[List[str]]:
         """Convert a checkbox completion to an answer."""
         assert question.question_type == "checkbox"
@@ -147,7 +147,9 @@ class ModelCritiqueClient(CritiqueClient):
             hlog(f"Error parsing answer: {e}. Skipping question (and so the respondent entirely)")
             return None
 
-    def _free_response_completion_to_answer(self, question: CritiqueQuestionTemplate, completion: Sequence) -> str:
+    def _free_response_completion_to_answer(
+        self, question: CritiqueQuestionTemplate, completion: GeneratedOutput
+    ) -> str:
         """Convert a free response completion to an answer."""
         assert question.question_type == "free_response"
         return completion.text

helm/proxy/example_queries.py

@@ -21,6 +21,8 @@ example_queries = [
             """
             temperature: 0.5  # Medium amount of randomness
             stop_sequences: [.]  # Stop when you hit a period
+            model: openai/gpt-3.5-turbo-0613
+            model_deployment: openai/gpt-3.5-turbo-0613
             """
         ),
         environments="",
@@ -31,7 +33,9 @@ example_queries = [
             """
             temperature: 0.5  # Medium amount of randomness
             stop_sequences: [\\n]  # Stop when you hit a newline
-            num_completions: 10  # Generate many samples
+            num_completions: 5  # Generate many samples
+            model: openai/gpt-3.5-turbo-0613
+            model_deployment: openai/gpt-3.5-turbo-0613
             """
         ),
         environments="",
@@ -42,7 +46,9 @@ example_queries = [
             """
             echo_prompt: true  # Analyze the prompt
             max_tokens: 0  # Don't generate any more
-            top_k_per_token: 10  # Show alternatives for each position
+            top_k_per_token: 5  # Show alternatives for each position
+            model: openai/davinci-002
+            model_deployment: openai/davinci-002
             """
         ),
         environments=dedent(""),
@@ -53,6 +59,8 @@ example_queries = [
             """
             temperature: 0  # Deterministic
             max_tokens: 50
+            model: openai/gpt-3.5-turbo-0613
+            model_deployment: openai/gpt-3.5-turbo-0613
             """
         ),
         environments="",
@@ -63,13 +71,15 @@ example_queries = [
             """
             temperature: 0
             stop_sequences: [.]
-            model_deployment: ${model_deployment}  # Try out multiple models
+            # Try out multiple models
+            model: ${model}
+            model_deployment: ${model}
             """
         ),
         environments=dedent(
             """
             occupation: [mathematician, lawyer, doctor]
-            model_deployment: [openai/davinci, ai21/j1-jumbo]
+            model: [openai/gpt-3.5-turbo-0613, openai/gpt-3.5-turbo-1106]
             """
         ),
     ),
@@ -88,12 +98,14 @@ example_queries = [
             temperature: 0.5
             stop_sequences: [\\n]
             num_completions: 5
-            model_deployment: ${model_deployment}  # Try out GPT-3 and Jurassic
+            # Try out multiple models
+            model: ${model}
+            model_deployment: ${model}
             """
         ),
         environments=dedent(
             """
-            model_deployment: [openai/davinci, ai21/j1-jumbo]
+            model: [openai/gpt-3.5-turbo-0613, openai/gpt-3.5-turbo-1106]
             """
         ),
     ),
@@ -122,20 +134,23 @@ example_queries = [
             temperature: 0
             max_tokens: 1
             top_k_per_token: 4
-            model_deployment: ${model_deployment}  # Try out GPT-3 and Jurassic
+            # Try out multiple models
+            model: ${model}
+            model_deployment: ${model}
             """
         ),
         environments=dedent(
             """
-            model_deployment: [openai/davinci, ai21/j1-jumbo]
+            model: [openai/gpt-3.5-turbo-0613, openai/gpt-3.5-turbo-1106]
             """
         ),
     ),
     Query(
-        prompt="Takes two vectors a and b and returns their Euclidean distance",
+        prompt="Write a Python function that takes two vectors a and b and returns their Euclidean distance.",
         settings=dedent(
             """
-            model_deployment: openai/code-davinci-001  # Codex for code generation
+            model: openai/gpt-3.5-turbo-0613
+            model_deployment: openai/gpt-3.5-turbo-0613
             """
         ),
         environments="",
@@ -144,19 +159,16 @@ example_queries = [
         prompt="The quick brown fox",
         settings=dedent(
             """
-            model_deployment: ${model_deployment}
             temperature: 0.3
             stop_sequences: [\\n]
+            # Try out multiple models
+            model: ${model}
+            model_deployment: ${model}
             """
         ),
         environments=dedent(
             """
-            model_deployment: [
-                "openai/davinci", "openai/text-davinci-002",
-                "openai/text-davinci-003", "ai21/j1-grande-v2-beta",
-                "together/gpt-j-6b", "together/gpt-jt-6b-v1",
-                "together/bloom", "together/opt-175b"
-            ]
+            model: [openai/gpt-3.5-turbo-0613, openai/gpt-3.5-turbo-1106]
             """
         ),
     ),