crfm-helm 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

helm/config/model_deployments.yaml

@@ -28,74 +28,47 @@ model_deployments:
 
   # AI21 Labs
 
-  # J1 models are Deprecated by AI21 Labs
-  # API returns: Detail: Jurassic J1 models are deprecated
-  - name: ai21/j1-jumbo
-    deprecated: true
-    model_name: ai21/j1-jumbo
-    tokenizer_name: ai21/j1
-    max_sequence_length: 2047
-    client_spec:
-      class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-
-  - name: ai21/j1-large
-    deprecated: true
-    model_name: ai21/j1-large
-    tokenizer_name: ai21/j1
-    max_sequence_length: 2047
-    client_spec:
-      class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-
-  - name: ai21/j1-grande
-    deprecated: true
-    model_name: ai21/j1-grande
-    tokenizer_name: ai21/j1
+  - name: ai21/j2-large
+    model_name: ai21/j2-large
+    tokenizer_name: ai21/j2-tokenizer
     max_sequence_length: 2047
     client_spec:
       class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
 
-  - name: ai21/j1-grande-v2-beta
-    deprecated: true
-    model_name: ai21/j1-grande-v2-beta
-    tokenizer_name: ai21/j1
+  - name: ai21/j2-grande
+    model_name: ai21/j2-grande
+    tokenizer_name: ai21/j2-tokenizer
     max_sequence_length: 2047
     client_spec:
       class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
 
   - name: ai21/j2-jumbo
     model_name: ai21/j2-jumbo
-    tokenizer_name: ai21/j1
+    tokenizer_name: ai21/j2-tokenizer
     max_sequence_length: 6000
     client_spec:
       class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
 
-  - name: ai21/j2-large
-    model_name: ai21/j2-large
-    tokenizer_name: ai21/j1
-    max_sequence_length: 2047
+  - name: ai21/jamba-instruct
+    model_name: ai21/jamba-instruct
+    tokenizer_name: ai21/jamba-instruct-tokenizer
+    max_sequence_length: 256000
     client_spec:
-      class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
+      class_name: "helm.clients.ai21_client.AI21ChatClient"
 
-  - name: ai21/j2-grande
-    model_name: ai21/j2-grande
-    tokenizer_name: ai21/j1
-    max_sequence_length: 2047
+  - name: ai21/jamba-1.5-mini
+    model_name: ai21/jamba-1.5-mini
+    tokenizer_name: ai21/jamba-1.5-mini-tokenizer
+    max_sequence_length: 256000
     client_spec:
-      class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
+      class_name: "helm.clients.ai21_client.AI21ChatClient"
+
+  - name: ai21/jamba-1.5-large
+    model_name: ai21/jamba-1.5-large
+    tokenizer_name: ai21/jamba-1.5-large-tokenizer
+    max_sequence_length: 256000
+    client_spec:
+      class_name: "helm.clients.ai21_client.AI21ChatClient"
 
   # Aleph Alpha
   - name: AlephAlpha/luminous-base
@@ -216,6 +189,13 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
 
+  - name: anthropic/claude-3-5-sonnet-20240620
+    model_name: anthropic/claude-3-5-sonnet-20240620
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
+
   - name: anthropic/stanford-online-all-v4-s3
     deprecated: true # Closed model, not accessible via API
     model_name: anthropic/stanford-online-all-v4-s3
@@ -225,105 +205,40 @@ model_deployments:
       class_name: "helm.clients.anthropic_client.AnthropicLegacyClient"
 
   # Cohere
-  - name: cohere/xlarge-20220609
-    model_name: cohere/xlarge-20220609
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2047
-    max_request_length: 2048
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/large-20220720
-    model_name: cohere/large-20220720
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2047
-    max_request_length: 2048
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/medium-20220720
-    model_name: cohere/medium-20220720
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2047
-    max_request_length: 2048
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/small-20220720
-    model_name: cohere/small-20220720
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2047
-    max_request_length: 2048
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/xlarge-20221108
-    model_name: cohere/xlarge-20221108
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2047
-    max_request_length: 2048
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/medium-20221108
-    model_name: cohere/medium-20221108
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2047
-    max_request_length: 2048
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/command-medium-beta
-    model_name: cohere/command-medium-beta
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2019
-    max_request_length: 2020
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/command-xlarge-beta
-    model_name: cohere/command-xlarge-beta
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2019
-    max_request_length: 2020
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
   - name: cohere/command
     model_name: cohere/command
-    tokenizer_name: cohere/cohere
+    tokenizer_name: cohere/command
     max_sequence_length: 2019 # TODO: verify this
     max_request_length: 2020 # TODO: verify this
     client_spec:
       class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
 
   - name: cohere/command-light
     model_name: cohere/command-light
-    tokenizer_name: cohere/cohere
+    tokenizer_name: cohere/command-light
     max_sequence_length: 2019 # TODO: verify this
     max_request_length: 2020 # TODO: verify this
     client_spec:
       class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
+
+  - name: cohere/command-r
+    model_name: cohere/command-r
+    tokenizer_name: cohere/command-r
+    max_sequence_length: 128000
+    max_request_length: 128000
+    client_spec:
+      class_name: "helm.clients.cohere_client.CohereChatClient"
+
+  - name: cohere/command-r-plus
+    model_name: cohere/command-r-plus
+    tokenizer_name: cohere/command-r-plus
+    # "We have a known issue where prompts between 112K - 128K in length
+    # result in bad generations."
+    # Source: https://docs.cohere.com/docs/command-r-plus
+    max_sequence_length: 110000
+    max_request_length: 110000
+    client_spec:
+      class_name: "helm.clients.cohere_client.CohereChatClient"
 
   # Craiyon
 
@@ -352,7 +267,7 @@ model_deployments:
     tokenizer_name: databricks/dbrx-instruct
     max_sequence_length: 32767
     client_spec:
-      class_name: "helm.clients.together_client.TogetherClient"
+      class_name: "helm.clients.together_client.TogetherChatClient"
 
   # DeepFloyd
 
@@ -390,7 +305,7 @@ model_deployments:
     tokenizer_name: deepseek-ai/deepseek-llm-67b-chat
     max_sequence_length: 4095
     client_spec:
-      class_name: "helm.clients.together_client.TogetherClient"
+      class_name: "helm.clients.together_client.TogetherChatClient"
 
   # Gooseai
 
@@ -434,6 +349,14 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.vertexai_client.VertexAIChatClient"
 
+  - name: google/gemini-1.0-pro-002
+    model_name: google/gemini-1.0-pro-002
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 30720
+    max_sequence_and_generated_tokens_length: 32768 # Officially max_sequence_length + 2048
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
   - name: google/gemini-pro-vision
     model_name: google/gemini-pro-vision
     tokenizer_name: openai/cl100k_base
@@ -450,6 +373,22 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.vertexai_client.VertexAIChatClient"
 
+  - name: google/gemini-1.5-flash-001
+    model_name: google/gemini-1.5-flash-001
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
+    # TODO: Max output tokens: 8192
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-1.5-pro-001
+    model_name: google/gemini-1.5-pro-001
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
+    # TODO: Max output tokens: 8192
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
   - name: google/gemini-1.5-pro-preview-0409
     model_name: google/gemini-1.5-pro-preview-0409
     tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
@@ -458,6 +397,63 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.vertexai_client.VertexAIChatClient"
 
+  - name: google/gemini-1.5-pro-preview-0514
+    model_name: google/gemini-1.5-pro-preview-0514
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
+    # TODO: Max output tokens: 8192
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-1.5-flash-preview-0514
+    model_name: google/gemini-1.5-flash-preview-0514
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
+    # TODO: Max output tokens: 8192
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  ## Gemini with different safety settings
+  - name: google/gemini-1.5-pro-001-safety-default
+    model_name: google/gemini-1.5-pro-001-safety-default
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
+    # TODO: Max output tokens: 8192
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+      args:
+        safety_settings_preset: default
+
+  - name: google/gemini-1.5-pro-001-safety-block-none
+    model_name: google/gemini-1.5-pro-001-safety-block-none
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
+    # TODO: Max output tokens: 8192
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+      args:
+        safety_settings_preset: block_none
+
+  - name: google/gemini-1.5-flash-001-safety-default
+    model_name: google/gemini-1.5-flash-001-safety-default
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
+    # TODO: Max output tokens: 8192
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+      args:
+        safety_settings_preset: default
+
+  - name: google/gemini-1.5-flash-001-safety-block-none
+    model_name: google/gemini-1.5-flash-001-safety-block-none
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
+    # TODO: Max output tokens: 8192
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+      args:
+        safety_settings_preset: block_none
+
   ## Gemma
   - name: together/gemma-2b
     model_name: google/gemma-2b
@@ -487,6 +483,56 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.together_client.TogetherClient"
 
+  - name: together/gemma-2-9b-it
+    model_name: google/gemma-2-9b-it
+    tokenizer_name: google/gemma-2-9b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/gemma-2-27b-it
+    model_name: google/gemma-2-27b-it
+    tokenizer_name: google/gemma-2-9b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  ## MedLM
+  - name: google/medlm-medium
+    model_name: google/medlm-medium
+    tokenizer_name: google/text-bison@001
+    max_sequence_length: 6000 # Officially 8192
+    max_sequence_and_generated_tokens_length: 7000 # Officially 9216
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
+
+  - name: google/medlm-large
+    model_name: google/medlm-large
+    tokenizer_name: google/text-bison@001
+    max_sequence_length: 6000 # Officially 8192
+    max_sequence_and_generated_tokens_length: 7000 # Officially 9216
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
+
+  ## PaliGemma
+  - name: google/paligemma-3b-mix-224
+    model_name: google/paligemma-3b-mix-224
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 7167
+    client_spec:
+      class_name: "helm.clients.vision_language.paligemma_client.PaliGemmaClient"
+
+  - name: google/paligemma-3b-mix-448
+    model_name: google/paligemma-3b-mix-448
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 7167
+    client_spec:
+      class_name: "helm.clients.vision_language.paligemma_client.PaliGemmaClient"
+
   ## PaLM 2
   - name: google/text-bison@001
     model_name: google/text-bison@001
@@ -504,7 +550,7 @@ model_deployments:
     max_sequence_length: 6000 # Officially 8192
     max_sequence_and_generated_tokens_length: 9216
     client_spec:
-      class_name: "helm.proxy.clients.vertexai_client.VertexAITextClient"
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
     window_service_spec:
       class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
 
@@ -545,7 +591,7 @@ model_deployments:
     max_sequence_length: 6000 # Officially 6144
     max_sequence_and_generated_tokens_length: 7168
     client_spec:
-      class_name: "helm.proxy.clients.vertexai_client.VertexAITextClient"
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
     window_service_spec:
       class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
 
@@ -561,6 +607,25 @@ model_deployments:
 
   # HuggingFace
 
+  ## AI Singapore
+  - name: huggingface/sea-lion-7b
+    model_name: aisingapore/sea-lion-7b
+    tokenizer_name: aisingapore/sea-lion-7b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        trust_remote_code: true
+
+  - name: huggingface/sea-lion-7b-instruct
+    model_name: aisingapore/sea-lion-7b-instruct
+    tokenizer_name: aisingapore/sea-lion-7b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        trust_remote_code: true
+
   ## Bigcode
   - name: huggingface/santacoder
     model_name: bigcode/santacoder
@@ -576,6 +641,15 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
 
+  ## Biomistral 
+
+  - name: huggingface/biomistral-7b
+    model_name: biomistral/biomistral-7b
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 32000
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
   ## Databricks
   - name: huggingface/dolly-v2-3b
     model_name: databricks/dolly-v2-3b
@@ -643,6 +717,47 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
 
+  ## Google
+  - name: huggingface/gemma-2-9b
+    model_name: google/gemma-2-9b
+    tokenizer_name: google/gemma-2-9b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+        torch_dtype: torch.bfloat16
+
+  - name: huggingface/gemma-2-9b-it
+    model_name: google/gemma-2-9b-it
+    tokenizer_name: google/gemma-2-9b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+        torch_dtype: torch.bfloat16
+
+  - name: huggingface/gemma-2-27b
+    model_name: google/gemma-2-27b
+    tokenizer_name: google/gemma-2-9b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+        torch_dtype: torch.bfloat16
+
+  - name: huggingface/gemma-2-27b-it
+    model_name: google/gemma-2-27b-it
+    tokenizer_name: google/gemma-2-9b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+        torch_dtype: torch.bfloat16
+
   ## LMSYS
   - name: huggingface/vicuna-7b-v1.3
     model_name: lmsys/vicuna-7b-v1.3
@@ -658,6 +773,15 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
 
+  ## Meditron 
+
+  - name: huggingface/meditron-7b
+    model_name: epfl-llm/meditron-7b
+    tokenizer_name: meta-llama/Llama-2-7b-hf
+    max_sequence_length: 4094
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
   ## Meta
   - name: huggingface/opt-175b
     model_name: meta/opt-175b
@@ -738,6 +862,14 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
 
+  ## KAIST AI
+  - name: huggingface/prometheus-vision-13b-v1.0-hf
+    model_name: kaistai/prometheus-vision-13b-v1.0-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+
   ## OpenFlamingo
   - name: openflamingo/OpenFlamingo-9B-vitl-mpt7b
     model_name: openflamingo/OpenFlamingo-9B-vitl-mpt7b
@@ -750,6 +882,7 @@ model_deployments:
         tokenizer_name: "anas-awadalla-2/mpt-7b"
         cross_attn_every_n_layers: 4
 
+  ## Microsoft
   - name: together/phi-2
     model_name: microsoft/phi-2
     tokenizer_name: microsoft/phi-2
@@ -757,6 +890,26 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.together_client.TogetherClient" 
 
+  - name: huggingface/phi-3-small-8k-instruct
+    model_name: microsoft/phi-3-small-8k-instruct
+    tokenizer_name: microsoft/phi-3-small-8k-instruct
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        torch_dtype: auto
+        trust_remote_code: true
+
+  - name: huggingface/phi-3-medium-4k-instruct
+    model_name: microsoft/phi-3-medium-4k-instruct
+    tokenizer_name: microsoft/phi-3-medium-4k-instruct
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+        torch_dtype: auto
+      
   ## Mistral AI
   - name: huggingface/bakLlava-v1-hf
     model_name: mistralai/bakLlava-v1-hf
@@ -811,6 +964,179 @@ model_deployments:
       args:
         pretrained_model_name_or_path: openai-community/gpt2
 
+  ## OpenThaiGPT
+  - name: huggingface/openthaigpt-1.0.0-7b-chat
+    model_name: openthaigpt/openthaigpt-1.0.0-7b-chat
+    tokenizer_name: openthaigpt/openthaigpt-1.0.0-7b-chat
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/openthaigpt-1.0.0-13b-chat
+    model_name: openthaigpt/openthaigpt-1.0.0-13b-chat
+    tokenizer_name: openthaigpt/openthaigpt-1.0.0-7b-chat
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  - name: huggingface/openthaigpt-1.0.0-70b-chat
+    model_name: openthaigpt/openthaigpt-1.0.0-70b-chat
+    tokenizer_name: huggingface/openthaigpt-1.0.0-7b-chat
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  ## SAIL (SEA AI Lab)
+  - name: huggingface/sailor-7b
+    model_name: sail/sailor-7b
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/sailor-7b-chat
+    model_name: sail/sailor-7b-chat
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/sailor-14b
+    model_name: sail/sailor-14b
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  - name: huggingface/sailor-14b-chat
+    model_name: sail/sailor-14b-chat
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  # SambaNova
+  - name: huggingface/sambalingo-thai-base
+    model_name: sambanova/sambalingo-thai-base
+    tokenizer_name: sambanova/sambalingo-thai-base
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base
+
+  - name: huggingface/sambalingo-thai-chat
+    model_name: sambanova/sambalingo-thai-chat
+    tokenizer_name: sambanova/sambalingo-thai-base
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base
+
+  - name: huggingface/sambalingo-thai-base-70b
+    model_name: sambanova/sambalingo-thai-base-70b
+    tokenizer_name: sambanova/sambalingo-thai-base
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base-70B
+        device_map: auto
+
+  - name: huggingface/sambalingo-thai-chat-70b
+    model_name: sambanova/sambalingo-thai-chat-70b
+    tokenizer_name: sambanova/sambalingo-thai-base
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base-70B
+        device_map: auto
+
+  ## SCB10X
+  - name: huggingface/typhoon-7b
+    model_name: scb10x/typhoon-7b
+    tokenizer_name: scb10x/typhoon-7b
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/typhoon-v1.5-8b
+    model_name: scb10x/typhoon-v1.5-8b
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/typhoon-v1.5-8b-instruct
+    model_name: scb10x/typhoon-v1.5-8b-instruct
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/typhoon-v1.5-72b
+    model_name: scb10x/typhoon-v1.5-72b
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  - name: huggingface/typhoon-v1.5-72b-instruct
+    model_name: scb10x/typhoon-v1.5-72b-instruct
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  - name: huggingface/llama-3-typhoon-v1.5x-8b-instruct
+    model_name: scb10x/llama-3-typhoon-v1.5x-8b-instruct
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/llama-3-typhoon-v1.5x-70b-instruct
+    model_name: scb10x/llama-3-typhoon-v1.5x-70b-instruct
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  # Alibaba DAMO Academy
+  - name: huggingface/seallm-7b-v2
+    model_name: damo/seallm-7b-v2
+    tokenizer_name: damo/seallm-7b-v2
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: SeaLLMs/SeaLLM-7B-v2
+
+  - name: huggingface/seallm-7b-v2.5
+    model_name: damo/seallm-7b-v2.5
+    tokenizer_name: damo/seallm-7b-v2.5
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: SeaLLMs/SeaLLM-7B-v2.5
+
   ## StabilityAI
   - name: huggingface/stablelm-base-alpha-3b
     model_name: stabilityai/stablelm-base-alpha-3b
@@ -1090,134 +1416,62 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.mistral_client.MistralAIClient"
 
-  # Neurips
-  - name: neurips/local
-    model_name: neurips/local
-    tokenizer_name: neurips/local
-    max_sequence_length: 2048
-    client_spec:
-      class_name: "helm.clients.http_model_client.HTTPModelClient"
-
-  # Nvidia
-  - name: nvidia/megatron-gpt2
-    model_name: nvidia/megatron-gpt2
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 1024
-    client_spec:
-      class_name: "helm.clients.megatron_client.MegatronClient"
-
-  # OpenAI
-
-  ## GPT 3 Models
-
-  - name: openai/davinci-002
-    model_name: openai/davinci-002
-    tokenizer_name: openai/cl100k_base
-    # Claimed sequence length is 16,384 tokens but we round down to 16,000 tokens
-    # to provide a margin of error.
-    max_sequence_length: 16000
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/babbage-002
-    model_name: openai/babbage-002
-    tokenizer_name: openai/cl100k_base
-    # Claimed sequence length is 16,384 tokens but we round down to 16,000 tokens
-    # to provide a margin of error.
-    max_sequence_length: 16000
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  # The list of models can be found here: https://beta.openai.com/docs/engines/gpt-3
-  # DEPRECATED: Announced on July 06 2023 that these models will be shut down on January 04 2024.
-
-  - name: openai/davinci
-    deprecated: true
-    model_name: openai/davinci
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/curie
-    deprecated: true
-    model_name: openai/curie
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: mistralai/mistral-large-2407
+    model_name: mistralai/mistral-large-2407
+    tokenizer_name: mistralai/Mistral-Large-Instruct-2407
+    max_sequence_length: 128000
     client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
+      class_name: "helm.clients.mistral_client.MistralAIClient"
 
-  - name: openai/babbage
-    deprecated: true
-    model_name: openai/babbage
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: mistralai/open-mistral-nemo-2407
+    model_name: mistralai/open-mistral-nemo-2407
+    tokenizer_name: mistralai/Mistral-Nemo-Base-2407
+    max_sequence_length: 128000
     client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
+      class_name: "helm.clients.mistral_client.MistralAIClient"
 
-  - name: openai/ada
-    deprecated: true
-    model_name: openai/ada
-    tokenizer_name: huggingface/gpt2
+  # Neurips
+  - name: neurips/local
+    model_name: neurips/local
+    tokenizer_name: neurips/local
     max_sequence_length: 2048
-    max_request_length: 2049
     client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
+      class_name: "helm.clients.http_model_client.HTTPModelClient"
 
-  - name: openai/text-davinci-003
-    deprecated: true
-    model_name: openai/text-davinci-003
+  # Nvidia
+  - name: nvidia/megatron-gpt2
+    model_name: nvidia/megatron-gpt2
     tokenizer_name: huggingface/gpt2
-    max_sequence_length: 4000
-    max_request_length: 4001
+    max_sequence_length: 1024
     client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
+      class_name: "helm.clients.megatron_client.MegatronClient"
 
-  - name: openai/text-davinci-002
-    deprecated: true
-    model_name: openai/text-davinci-002
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 4000
-    max_request_length: 4001
+  - name: nvidia/nemotron-4-340b-instruct
+    model_name: nvidia/nemotron-4-340b-instruct
+    tokenizer_name: nvidia/nemotron-4-340b-instruct
+    max_sequence_length: 4085
     client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
+      class_name: "helm.clients.nvidia_nim_client.NvidiaNimClient"
 
-  - name: openai/text-davinci-001
-    deprecated: true
-    model_name: openai/text-davinci-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
+  # OpenAI
 
-  - name: openai/text-curie-001
-    deprecated: true
-    model_name: openai/text-curie-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
+  ## GPT 3 Models
 
-  - name: openai/text-babbage-001
-    deprecated: true
-    model_name: openai/text-babbage-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: openai/davinci-002
+    model_name: openai/davinci-002
+    tokenizer_name: openai/cl100k_base
+    # Claimed sequence length is 16,384 tokens but we round down to 16,000 tokens
+    # to provide a margin of error.
+    max_sequence_length: 16000
     client_spec:
       class_name: "helm.clients.openai_client.OpenAIClient"
 
-  - name: openai/text-ada-001
-    deprecated: true
-    model_name: openai/text-ada-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: openai/babbage-002
+    model_name: openai/babbage-002
+    tokenizer_name: openai/cl100k_base
+    # Claimed sequence length is 16,384 tokens but we round down to 16,000 tokens
+    # to provide a margin of error.
+    max_sequence_length: 16000
     client_spec:
       class_name: "helm.clients.openai_client.OpenAIClient"
 
@@ -1348,6 +1602,27 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.openai_client.OpenAIClient"
 
+  - name: openai/gpt-4o-2024-05-13
+    model_name: openai/gpt-4o-2024-05-13
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4o-2024-08-06
+    model_name: openai/gpt-4o-2024-08-06
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4o-mini-2024-07-18
+    model_name: openai/gpt-4o-mini-2024-07-18
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
   - name: openai/gpt-4-vision-preview
     model_name: openai/gpt-4-vision-preview
     tokenizer_name: openai/cl100k_base
@@ -1366,78 +1641,10 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.openai_client.OpenAIClient"
 
-  ## Codex Models
-  # DEPRECATED: Codex models have been shut down on March 23 2023.
-
-  - name: openai/code-davinci-002
-    deprecated: true
-    model_name: openai/code-davinci-002
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 4000
-    max_request_length: 4001
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/code-davinci-001
-    deprecated: true
-    model_name: openai/code-davinci-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/code-cushman-001
-    deprecated: true
-    model_name: openai/code-cushman-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
   ## Text Similarity Models
   # OpenAI similarity embedding models: https://beta.openai.com/docs/guides/embeddings
   # The number of parameters is guessed based on the number of parameters of the
   # corresponding GPT-3 model.
-  # DEPRECATED: Announced on July 06 2023 that first generation embeddings models
-  #  will be shut down on January 04 2024.
-
-  - name: openai/text-similarity-davinci-001
-    deprecated: true
-    model_name: openai/text-similarity-davinci-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/text-similarity-curie-001
-    deprecated: true
-    model_name: openai/text-similarity-curie-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/text-similarity-babbage-001
-    deprecated: true
-    model_name: openai/text-similarity-babbage-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/text-similarity-ada-001
-    deprecated: true
-    model_name: openai/text-similarity-ada-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
 
   # As of 2023-11-07, text-embedding-ada-002 is not deprecated:
   # "We recommend using text-embedding-ada-002 for nearly all use cases."
@@ -1635,23 +1842,77 @@ model_deployments:
       args:
         together_model: meta-llama/Meta-Llama-3-70B
 
+  - name: together/llama-3.1-8b-instruct-turbo
+    model_name: meta/llama-3.1-8b-instruct-turbo
+    tokenizer_name: meta/llama-3.1-8b
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
+
+  - name: together/llama-3.1-70b-instruct-turbo
+    model_name: meta/llama-3.1-70b-instruct-turbo
+    tokenizer_name: meta/llama-3.1-8b
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo
+
+  - name: together/llama-3.1-405b-instruct-turbo
+    model_name: meta/llama-3.1-405b-instruct-turbo
+    tokenizer_name: meta/llama-3.1-8b
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo
+
   - name: together/llama-3-8b-chat
     model_name: meta/llama-3-8b-chat
     tokenizer_name: meta/llama-3-8b
-    max_sequence_length: 8191
+    max_sequence_length: 8182
     client_spec:
-      class_name: "helm.clients.together_client.TogetherClient"
+      class_name: "helm.clients.together_client.TogetherChatClient"
       args:
-        together_model: meta-llama/Meta-Llama-3-8B
+        together_model: meta-llama/Llama-3-8b-chat-hf
 
   - name: together/llama-3-70b-chat
     model_name: meta/llama-3-70b-chat
     tokenizer_name: meta/llama-3-8b
-    max_sequence_length: 8191
+    max_sequence_length: 8182
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        together_model: meta-llama/Llama-3-70b-chat-hf
+
+  - name: together/llama-guard-7b
+    model_name: meta/llama-guard-7b
+    tokenizer_name: meta-llama/Llama-2-7b-hf
+    max_sequence_length: 2047
     client_spec:
       class_name: "helm.clients.together_client.TogetherClient"
       args:
-        together_model: meta-llama/Meta-Llama-3-70B
+        together_model: meta-llama/llama-guard-7b
+
+  - name: together/llama-guard-2-8b
+    model_name: meta/llama-guard-2-8b
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 4094
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/llamaguard-2-8b
+
+  - name: together/llama-guard-3-8b
+    model_name: meta/llama-guard-3-8b
+    tokenizer_name: meta/llama-3.1-8b
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-Guard-3-8B
 
   # 01.AI
   - name: together/yi-6b
@@ -1677,18 +1938,32 @@ model_deployments:
     tokenizer_name: 01-ai/Yi-6B
     max_sequence_length: 4095
     client_spec:
-      class_name: "helm.clients.together_client.TogetherClient"
+      class_name: "helm.clients.together_client.TogetherChatClient"
       args:
-        together_model: zero-one-ai/Yi-6B
+        together_model: zero-one-ai/Yi-6B-Chat
 
   - name: together/yi-34b-chat
     model_name: 01-ai/yi-34b-chat
     tokenizer_name: 01-ai/Yi-6B
     max_sequence_length: 4095
     client_spec:
-      class_name: "helm.clients.together_client.TogetherClient"
+      class_name: "helm.clients.together_client.TogetherChatClient"
       args:
-        together_model: zero-one-ai/Yi-34B
+        together_model: zero-one-ai/Yi-34B-Chat
+
+  - name: 01-ai/yi-large
+    model_name: 01-ai/yi-large
+    tokenizer_name: 01-ai/Yi-6B  # Actual tokenizer is publicly unavailable, so use a substitute
+    max_sequence_length: 16000
+    client_spec:
+      class_name: "helm.clients.yi_client.YiChatClient"
+
+  - name: 01-ai/yi-large-preview
+    model_name: 01-ai/yi-large-preview
+    tokenizer_name: 01-ai/Yi-6B  # Actual tokenizer is publicly unavailable, so use a substitute
+    max_sequence_length: 16000
+    client_spec:
+      class_name: "helm.clients.yi_client.YiChatClient"
 
 
   # Allen Institute for AI
@@ -1711,8 +1986,16 @@ model_deployments:
     tokenizer_name: allenai/olmo-7b
     max_sequence_length: 2047
     client_spec:
-      class_name: "helm.clients.together_client.TogetherClient"
+      class_name: "helm.clients.together_client.TogetherChatClient"
 
+  - name: huggingface/olmo-1.7-7b
+    model_name: allenai/olmo-1.7-7b
+    tokenizer_name: allenai/OLMo-1.7-7B-hf
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: allenai/OLMo-1.7-7B-hf
 
   ## MistralAI
   - name: together/mistral-7b-v0.1
@@ -1724,6 +2007,27 @@ model_deployments:
       args:
         together_model: mistralai/Mistral-7B-v0.1
 
+  - name: together/mistral-7b-instruct-v0.1
+    model_name: mistralai/mistral-7b-instruct-v0.1
+    tokenizer_name: mistralai/Mistral-7B-Instruct-v0.1
+    max_sequence_length: 4000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+
+  - name: together/mistral-7b-instruct-v0.2
+    model_name: mistralai/mistral-7b-instruct-v0.2
+    tokenizer_name: mistralai/Mistral-7B-Instruct-v0.2
+    max_sequence_length: 32000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+
+  - name: together/mistral-7b-instruct-v0.3
+    model_name: mistralai/mistral-7b-instruct-v0.3
+    tokenizer_name: mistralai/Mistral-7B-Instruct-v0.3
+    max_sequence_length: 32000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+
   - name: together/mixtral-8x7b-32kseqlen
     model_name: mistralai/mixtral-8x7b-32kseqlen
     tokenizer_name: mistralai/Mistral-7B-v0.1
@@ -1736,9 +2040,9 @@ model_deployments:
   - name: together/mixtral-8x7b-instruct-v0.1
     model_name: mistralai/mixtral-8x7b-instruct-v0.1
     tokenizer_name: mistralai/Mistral-7B-v0.1
-    max_sequence_length: 4095 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    max_sequence_length: 32767
     client_spec:
-      class_name: "helm.clients.together_client.TogetherClient"
+      class_name: "helm.clients.together_client.TogetherChatClient"
 
   - name: together/mixtral-8x22b
     model_name: mistralai/mixtral-8x22b
@@ -1752,7 +2056,7 @@ model_deployments:
     tokenizer_name: mistralai/Mistral-7B-v0.1
     max_sequence_length: 65535
     client_spec:
-      class_name: "helm.clients.together_client.TogetherClient"
+      class_name: "helm.clients.together_client.TogetherChatClient"
 
 
   ## Snowflake
@@ -1872,18 +2176,6 @@ model_deployments:
       args:
         together_model: togethercomputer/RedPajama-INCITE-7B-Instruct
 
-  ## Tsinghua
-  - name: together/glm
-    deprecated: true  # Removed from Together
-    model_name: tsinghua/glm
-    tokenizer_name: TsinghuaKEG/ice
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.together_client.TogetherClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ice_window_service.ICEWindowService"
-
   - name: thudm/cogview2
     model_name: thudm/cogview2
     tokenizer_name: openai/clip-vit-large-patch14
@@ -1922,22 +2214,6 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.palmyra_client.PalmyraClient"
 
-  - name: writer/palmyra-instruct-30
-    model_name: writer/palmyra-instruct-30
-    tokenizer_name: writer/gpt2
-    max_sequence_length: 2048
-    max_sequence_and_generated_tokens_length: 2048
-    client_spec:
-      class_name: "helm.clients.palmyra_client.PalmyraClient"
-
-  - name: writer/palmyra-e
-    model_name: writer/palmyra-e
-    tokenizer_name: writer/gpt2
-    max_sequence_length: 2048
-    max_sequence_and_generated_tokens_length: 2048
-    client_spec:
-      class_name: "helm.clients.palmyra_client.PalmyraClient"
-
   - name: writer/silk-road
     model_name: writer/silk-road
     tokenizer_name: writer/gpt2
@@ -1978,12 +2254,21 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.palmyra_client.PalmyraClient"
 
+  - name: writer/palmyra-vision-003
+    model_name: writer/palmyra-vision-003
+    tokenizer_name: writer/gpt2
+    max_sequence_length: 2048
+    max_sequence_and_generated_tokens_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.palmyra_vision_client.PalmyraVisionClient"
+
+
   # Qwen
 
   - name: together/qwen-7b
     model_name: qwen/qwen-7b
     tokenizer_name: qwen/qwen-7b
-    max_sequence_length: 8191
+    max_sequence_length: 32767
     client_spec:
       class_name: "helm.clients.together_client.TogetherClient"
       args:
@@ -2019,12 +2304,54 @@ model_deployments:
   - name: together/qwen1.5-72b
     model_name: qwen/qwen1.5-72b
     tokenizer_name: qwen/qwen1.5-7b
-    max_sequence_length: 4095
+    max_sequence_length: 32767
     client_spec:
       class_name: "helm.clients.together_client.TogetherClient"
       args:
         together_model: Qwen/Qwen1.5-72B
 
+  - name: together/qwen1.5-7b-chat
+    model_name: qwen/qwen1.5-7b-chat
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+
+  - name: together/qwen1.5-14b-chat
+    model_name: qwen/qwen1.5-14b-chat
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+
+  - name: together/qwen1.5-32b-chat
+    model_name: qwen/qwen1.5-32b-chat
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+
+  - name: together/qwen1.5-72b-chat
+    model_name: qwen/qwen1.5-72b-chat
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+
+  - name: together/qwen1.5-110b-chat
+    model_name: qwen/qwen1.5-110b-chat
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+
+  - name: together/qwen2-72b-instruct
+    model_name: qwen/qwen2-72b-instruct
+    tokenizer_name: qwen/qwen2-72b-instruct
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+
   - name: huggingface/qwen-vl
     model_name: qwen/qwen-vl
     tokenizer_name: qwen/qwen-vl
@@ -2038,3 +2365,53 @@ model_deployments:
     max_sequence_length: 8191
     client_spec:
       class_name: "helm.clients.vision_language.qwen_vlm_client.QwenVLMClient"
+
+# Reka
+  - name: reka/reka-core
+    model_name: reka/reka-core
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.reka_client.RekaClient"
+  
+  - name: reka/reka-core-20240415
+    model_name: reka/reka-core-20240415
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.reka_client.RekaClient"
+  
+  - name: reka/reka-core-20240501
+    model_name: reka/reka-core-20240501
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.reka_client.RekaClient"
+
+  - name: reka/reka-flash
+    model_name: reka/reka-flash
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.reka_client.RekaClient"
+
+  - name: reka/reka-flash-20240226
+    model_name: reka/reka-flash-20240226
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.reka_client.RekaClient"
+
+  - name: reka/reka-edge
+    model_name: reka/reka-edge
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 64000
+    client_spec:
+      class_name: "helm.clients.reka_client.RekaClient"
+
+  - name: reka/reka-edge-20240208
+    model_name: reka/reka-edge-20240208
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 64000
+    client_spec:
+      class_name: "helm.clients.reka_client.RekaClient"