crfm-helm 0.5.2__py3-none-any.whl → 0.5.4__py3-none-any.whl

helm/config/model_deployments.yaml

@@ -28,74 +28,47 @@ model_deployments:
 
   # AI21 Labs
 
-  # J1 models are Deprecated by AI21 Labs
-  # API returns: Detail: Jurassic J1 models are deprecated
-  - name: ai21/j1-jumbo
-    deprecated: true
-    model_name: ai21/j1-jumbo
-    tokenizer_name: ai21/j1
-    max_sequence_length: 2047
-    client_spec:
-      class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-
-  - name: ai21/j1-large
-    deprecated: true
-    model_name: ai21/j1-large
-    tokenizer_name: ai21/j1
-    max_sequence_length: 2047
-    client_spec:
-      class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-
-  - name: ai21/j1-grande
-    deprecated: true
-    model_name: ai21/j1-grande
-    tokenizer_name: ai21/j1
+  - name: ai21/j2-large
+    model_name: ai21/j2-large
+    tokenizer_name: ai21/j2-tokenizer
     max_sequence_length: 2047
     client_spec:
       class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
 
-  - name: ai21/j1-grande-v2-beta
-    deprecated: true
-    model_name: ai21/j1-grande-v2-beta
-    tokenizer_name: ai21/j1
+  - name: ai21/j2-grande
+    model_name: ai21/j2-grande
+    tokenizer_name: ai21/j2-tokenizer
     max_sequence_length: 2047
     client_spec:
       class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
 
   - name: ai21/j2-jumbo
     model_name: ai21/j2-jumbo
-    tokenizer_name: ai21/j1
+    tokenizer_name: ai21/j2-tokenizer
     max_sequence_length: 6000
     client_spec:
       class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
 
-  - name: ai21/j2-large
-    model_name: ai21/j2-large
-    tokenizer_name: ai21/j1
-    max_sequence_length: 2047
+  - name: ai21/jamba-instruct
+    model_name: ai21/jamba-instruct
+    tokenizer_name: ai21/jamba-instruct-tokenizer
+    max_sequence_length: 256000
     client_spec:
-      class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
+      class_name: "helm.clients.ai21_client.AI21ChatClient"
 
-  - name: ai21/j2-grande
-    model_name: ai21/j2-grande
-    tokenizer_name: ai21/j1
-    max_sequence_length: 2047
+  - name: ai21/jamba-1.5-mini
+    model_name: ai21/jamba-1.5-mini
+    tokenizer_name: ai21/jamba-1.5-mini-tokenizer
+    max_sequence_length: 256000
     client_spec:
-      class_name: "helm.clients.ai21_client.AI21Client"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
+      class_name: "helm.clients.ai21_client.AI21ChatClient"
+
+  - name: ai21/jamba-1.5-large
+    model_name: ai21/jamba-1.5-large
+    tokenizer_name: ai21/jamba-1.5-large-tokenizer
+    max_sequence_length: 256000
+    client_spec:
+      class_name: "helm.clients.ai21_client.AI21ChatClient"
 
   # Aleph Alpha
   - name: AlephAlpha/luminous-base
@@ -216,6 +189,13 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
 
+  - name: anthropic/claude-3-5-sonnet-20240620
+    model_name: anthropic/claude-3-5-sonnet-20240620
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
+
   - name: anthropic/stanford-online-all-v4-s3
     deprecated: true # Closed model, not accessible via API
     model_name: anthropic/stanford-online-all-v4-s3
@@ -225,86 +205,6 @@ model_deployments:
       class_name: "helm.clients.anthropic_client.AnthropicLegacyClient"
 
   # Cohere
-  - name: cohere/xlarge-20220609
-    model_name: cohere/xlarge-20220609
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2047
-    max_request_length: 2048
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/large-20220720
-    model_name: cohere/large-20220720
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2047
-    max_request_length: 2048
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/medium-20220720
-    model_name: cohere/medium-20220720
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2047
-    max_request_length: 2048
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/small-20220720
-    model_name: cohere/small-20220720
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2047
-    max_request_length: 2048
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/xlarge-20221108
-    model_name: cohere/xlarge-20221108
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2047
-    max_request_length: 2048
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/medium-20221108
-    model_name: cohere/medium-20221108
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2047
-    max_request_length: 2048
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/command-medium-beta
-    model_name: cohere/command-medium-beta
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2019
-    max_request_length: 2020
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
-  - name: cohere/command-xlarge-beta
-    model_name: cohere/command-xlarge-beta
-    tokenizer_name: cohere/cohere
-    max_sequence_length: 2019
-    max_request_length: 2020
-    client_spec:
-      class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-
   - name: cohere/command
     model_name: cohere/command
     tokenizer_name: cohere/command
@@ -312,8 +212,6 @@ model_deployments:
     max_request_length: 2020 # TODO: verify this
     client_spec:
       class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
 
   - name: cohere/command-light
     model_name: cohere/command-light
@@ -322,8 +220,6 @@ model_deployments:
     max_request_length: 2020 # TODO: verify this
     client_spec:
       class_name: "helm.clients.cohere_client.CohereClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
 
   - name: cohere/command-r
     model_name: cohere/command-r
@@ -587,6 +483,20 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.together_client.TogetherClient"
 
+  - name: together/gemma-2-9b-it
+    model_name: google/gemma-2-9b-it
+    tokenizer_name: google/gemma-2-9b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/gemma-2-27b-it
+    model_name: google/gemma-2-27b-it
+    tokenizer_name: google/gemma-2-9b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
   ## MedLM
   - name: google/medlm-medium
     model_name: google/medlm-medium
@@ -716,6 +626,26 @@ model_deployments:
       args:
         trust_remote_code: true
 
+  - name: huggingface/llama3-8b-cpt-sea-lionv2-base
+    model_name: aisingapore/llama3-8b-cpt-sea-lionv2-base
+    tokenizer_name: meta/llama-3-8b-instruct
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+        torch_dtype: torch.bfloat16
+
+  - name: huggingface/llama3-8b-cpt-sea-lionv2.1-instruct
+    model_name: aisingapore/llama3-8b-cpt-sea-lionv2.1-instruct
+    tokenizer_name: meta/llama-3-8b-instruct
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+        torch_dtype: torch.bfloat16
+
   ## Bigcode
   - name: huggingface/santacoder
     model_name: bigcode/santacoder
@@ -807,6 +737,47 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
 
+  ## Google
+  - name: huggingface/gemma-2-9b
+    model_name: google/gemma-2-9b
+    tokenizer_name: google/gemma-2-9b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+        torch_dtype: torch.bfloat16
+
+  - name: huggingface/gemma-2-9b-it
+    model_name: google/gemma-2-9b-it
+    tokenizer_name: google/gemma-2-9b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+        torch_dtype: torch.bfloat16
+
+  - name: huggingface/gemma-2-27b
+    model_name: google/gemma-2-27b
+    tokenizer_name: google/gemma-2-9b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+        torch_dtype: torch.bfloat16
+
+  - name: huggingface/gemma-2-27b-it
+    model_name: google/gemma-2-27b-it
+    tokenizer_name: google/gemma-2-9b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+        torch_dtype: torch.bfloat16
+
   ## LMSYS
   - name: huggingface/vicuna-7b-v1.3
     model_name: lmsys/vicuna-7b-v1.3
@@ -931,6 +902,7 @@ model_deployments:
         tokenizer_name: "anas-awadalla-2/mpt-7b"
         cross_attn_every_n_layers: 4
 
+  ## Microsoft
   - name: together/phi-2
     model_name: microsoft/phi-2
     tokenizer_name: microsoft/phi-2
@@ -938,6 +910,26 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.together_client.TogetherClient" 
 
+  - name: huggingface/phi-3-small-8k-instruct
+    model_name: microsoft/phi-3-small-8k-instruct
+    tokenizer_name: microsoft/phi-3-small-8k-instruct
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        torch_dtype: auto
+        trust_remote_code: true
+
+  - name: huggingface/phi-3-medium-4k-instruct
+    model_name: microsoft/phi-3-medium-4k-instruct
+    tokenizer_name: microsoft/phi-3-medium-4k-instruct
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+        torch_dtype: auto
+      
   ## Mistral AI
   - name: huggingface/bakLlava-v1-hf
     model_name: mistralai/bakLlava-v1-hf
@@ -992,42 +984,134 @@ model_deployments:
       args:
         pretrained_model_name_or_path: openai-community/gpt2
 
+  ## OpenThaiGPT
+  - name: huggingface/openthaigpt-1.0.0-7b-chat
+    model_name: openthaigpt/openthaigpt-1.0.0-7b-chat
+    tokenizer_name: openthaigpt/openthaigpt-1.0.0-7b-chat
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/openthaigpt-1.0.0-13b-chat
+    model_name: openthaigpt/openthaigpt-1.0.0-13b-chat
+    tokenizer_name: openthaigpt/openthaigpt-1.0.0-7b-chat
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  - name: huggingface/openthaigpt-1.0.0-70b-chat
+    model_name: openthaigpt/openthaigpt-1.0.0-70b-chat
+    tokenizer_name: huggingface/openthaigpt-1.0.0-7b-chat
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
   ## SAIL (SEA AI Lab)
-  - name: sail/sailor-7b
+  - name: huggingface/sailor-7b
     model_name: sail/sailor-7b
     tokenizer_name: qwen/qwen1.5-7b
     max_sequence_length: 32768
     client_spec:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
 
-  - name: sail/sailor-7b-chat
+  - name: huggingface/sailor-7b-chat
     model_name: sail/sailor-7b-chat
     tokenizer_name: qwen/qwen1.5-7b
     max_sequence_length: 32768
     client_spec:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
 
-  - name: sail/sailor-14b
+  - name: huggingface/sailor-14b
     model_name: sail/sailor-14b
     tokenizer_name: qwen/qwen1.5-7b
     max_sequence_length: 32768
     client_spec:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
 
-  - name: sail/sailor-14b-chat
+  - name: huggingface/sailor-14b-chat
     model_name: sail/sailor-14b-chat
     tokenizer_name: qwen/qwen1.5-7b
     max_sequence_length: 32768
     client_spec:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  # SambaNova
+  - name: huggingface/sambalingo-thai-base
+    model_name: sambanova/sambalingo-thai-base
+    tokenizer_name: sambanova/sambalingo-thai-base
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base
+
+  - name: huggingface/sambalingo-thai-chat
+    model_name: sambanova/sambalingo-thai-chat
+    tokenizer_name: sambanova/sambalingo-thai-base
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base
+
+  - name: huggingface/sambalingo-thai-base-70b
+    model_name: sambanova/sambalingo-thai-base-70b
+    tokenizer_name: sambanova/sambalingo-thai-base
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base-70B
+        device_map: auto
+
+  - name: huggingface/sambalingo-thai-chat-70b
+    model_name: sambanova/sambalingo-thai-chat-70b
+    tokenizer_name: sambanova/sambalingo-thai-base
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: sambanovasystems/SambaLingo-Thai-Base-70B
+        device_map: auto
 
   ## SCB10X
+  - name: huggingface/typhoon-7b
+    model_name: scb10x/typhoon-7b
+    tokenizer_name: scb10x/typhoon-7b
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/typhoon-v1.5-8b
+    model_name: scb10x/typhoon-v1.5-8b
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/typhoon-v1.5-8b-instruct
+    model_name: scb10x/typhoon-v1.5-8b-instruct
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
   - name: huggingface/typhoon-v1.5-72b
     model_name: scb10x/typhoon-v1.5-72b
     tokenizer_name: qwen/qwen1.5-7b
     max_sequence_length: 32768
     client_spec:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
 
   - name: huggingface/typhoon-v1.5-72b-instruct
     model_name: scb10x/typhoon-v1.5-72b-instruct
@@ -1035,6 +1119,43 @@ model_deployments:
     max_sequence_length: 32768
     client_spec:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  - name: huggingface/llama-3-typhoon-v1.5x-8b-instruct
+    model_name: scb10x/llama-3-typhoon-v1.5x-8b-instruct
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/llama-3-typhoon-v1.5x-70b-instruct
+    model_name: scb10x/llama-3-typhoon-v1.5x-70b-instruct
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  # Alibaba DAMO Academy
+  - name: huggingface/seallm-7b-v2
+    model_name: damo/seallm-7b-v2
+    tokenizer_name: damo/seallm-7b-v2
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: SeaLLMs/SeaLLM-7B-v2
+
+  - name: huggingface/seallm-7b-v2.5
+    model_name: damo/seallm-7b-v2.5
+    tokenizer_name: damo/seallm-7b-v2.5
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: SeaLLMs/SeaLLM-7B-v2.5
 
   ## StabilityAI
   - name: huggingface/stablelm-base-alpha-3b
@@ -1315,6 +1436,20 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.mistral_client.MistralAIClient"
 
+  - name: mistralai/mistral-large-2407
+    model_name: mistralai/mistral-large-2407
+    tokenizer_name: mistralai/Mistral-Large-Instruct-2407
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.mistral_client.MistralAIClient"
+
+  - name: mistralai/open-mistral-nemo-2407
+    model_name: mistralai/open-mistral-nemo-2407
+    tokenizer_name: mistralai/Mistral-Nemo-Base-2407
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.mistral_client.MistralAIClient"
+
   # Neurips
   - name: neurips/local
     model_name: neurips/local
@@ -1331,6 +1466,13 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.megatron_client.MegatronClient"
 
+  - name: nvidia/nemotron-4-340b-instruct
+    model_name: nvidia/nemotron-4-340b-instruct
+    tokenizer_name: nvidia/nemotron-4-340b-instruct
+    max_sequence_length: 4085
+    client_spec:
+      class_name: "helm.clients.nvidia_nim_client.NvidiaNimClient"
+
   # OpenAI
 
   ## GPT 3 Models
@@ -1353,99 +1495,6 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.openai_client.OpenAIClient"
 
-  # The list of models can be found here: https://beta.openai.com/docs/engines/gpt-3
-  # DEPRECATED: Announced on July 06 2023 that these models will be shut down on January 04 2024.
-
-  - name: openai/davinci
-    deprecated: true
-    model_name: openai/davinci
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/curie
-    deprecated: true
-    model_name: openai/curie
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/babbage
-    deprecated: true
-    model_name: openai/babbage
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/ada
-    deprecated: true
-    model_name: openai/ada
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/text-davinci-003
-    deprecated: true
-    model_name: openai/text-davinci-003
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 4000
-    max_request_length: 4001
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/text-davinci-002
-    deprecated: true
-    model_name: openai/text-davinci-002
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 4000
-    max_request_length: 4001
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/text-davinci-001
-    deprecated: true
-    model_name: openai/text-davinci-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/text-curie-001
-    deprecated: true
-    model_name: openai/text-curie-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/text-babbage-001
-    deprecated: true
-    model_name: openai/text-babbage-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/text-ada-001
-    deprecated: true
-    model_name: openai/text-ada-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
   ## GPT 3.5 Turbo Models
   # ChatGPT: https://openai.com/blog/chatgpt
 
@@ -1580,6 +1629,20 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.openai_client.OpenAIClient"
 
+  - name: openai/gpt-4o-2024-08-06
+    model_name: openai/gpt-4o-2024-08-06
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4o-mini-2024-07-18
+    model_name: openai/gpt-4o-mini-2024-07-18
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
   - name: openai/gpt-4-vision-preview
     model_name: openai/gpt-4-vision-preview
     tokenizer_name: openai/cl100k_base
@@ -1598,33 +1661,18 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.openai_client.OpenAIClient"
 
-  ## Codex Models
-  # DEPRECATED: Codex models have been shut down on March 23 2023.
-
-  - name: openai/code-davinci-002
-    deprecated: true
-    model_name: openai/code-davinci-002
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 4000
-    max_request_length: 4001
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/code-davinci-001
-    deprecated: true
-    model_name: openai/code-davinci-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
+  ## o1 Models
+  - name: openai/o1-preview-2024-09-12
+    model_name: openai/o1-preview-2024-09-12
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
     client_spec:
       class_name: "helm.clients.openai_client.OpenAIClient"
 
-  - name: openai/code-cushman-001
-    deprecated: true
-    model_name: openai/code-cushman-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: openai/o1-mini-2024-09-12
+    model_name: openai/o1-mini-2024-09-12
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
     client_spec:
       class_name: "helm.clients.openai_client.OpenAIClient"
 
@@ -1632,44 +1680,6 @@ model_deployments:
   # OpenAI similarity embedding models: https://beta.openai.com/docs/guides/embeddings
   # The number of parameters is guessed based on the number of parameters of the
   # corresponding GPT-3 model.
-  # DEPRECATED: Announced on July 06 2023 that first generation embeddings models
-  #  will be shut down on January 04 2024.
-
-  - name: openai/text-similarity-davinci-001
-    deprecated: true
-    model_name: openai/text-similarity-davinci-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/text-similarity-curie-001
-    deprecated: true
-    model_name: openai/text-similarity-curie-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/text-similarity-babbage-001
-    deprecated: true
-    model_name: openai/text-similarity-babbage-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-
-  - name: openai/text-similarity-ada-001
-    deprecated: true
-    model_name: openai/text-similarity-ada-001
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
 
   # As of 2023-11-07, text-embedding-ada-002 is not deprecated:
   # "We recommend using text-embedding-ada-002 for nearly all use cases."
@@ -1856,7 +1866,25 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.together_client.TogetherClient"
       args:
-        together_model: meta-llama/Meta-Llama-3-8B
+        together_model: meta-llama/Llama-3-8b-hf
+
+  - name: together/llama-3-8b-instruct-turbo
+    model_name: meta/llama-3-8b-instruct-turbo
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3-8B-Instruct-Turbo
+
+  - name: together/llama-3-8b-instruct-lite
+    model_name: meta/llama-3-8b-instruct-lite
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3-8B-Instruct-Lite
 
   - name: together/llama-3-70b
     model_name: meta/llama-3-70b
@@ -1867,10 +1895,55 @@ model_deployments:
       args:
         together_model: meta-llama/Meta-Llama-3-70B
 
-  - name: together/llama-3-8b-chat
-    model_name: meta/llama-3-8b-chat
+  - name: together/llama-3-70b-instruct-turbo
+    model_name: meta/llama-3-70b-instruct-turbo
     tokenizer_name: meta/llama-3-8b
     max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3-70B-Instruct-Turbo
+
+  - name: together/llama-3-70b-instruct-lite
+    model_name: meta/llama-3-70b-instruct-lite
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3-70B-Instruct-Lite
+
+  - name: together/llama-3.1-8b-instruct-turbo
+    model_name: meta/llama-3.1-8b-instruct-turbo
+    tokenizer_name: meta/llama-3.1-8b
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo
+
+  - name: together/llama-3.1-70b-instruct-turbo
+    model_name: meta/llama-3.1-70b-instruct-turbo
+    tokenizer_name: meta/llama-3.1-8b
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo
+
+  - name: together/llama-3.1-405b-instruct-turbo
+    model_name: meta/llama-3.1-405b-instruct-turbo
+    tokenizer_name: meta/llama-3.1-8b
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo
+
+  - name: together/llama-3-8b-chat
+    model_name: meta/llama-3-8b-chat
+    tokenizer_name: meta/llama-3-8b-instruct
+    max_sequence_length: 8182
     client_spec:
       class_name: "helm.clients.together_client.TogetherChatClient"
       args:
@@ -1878,13 +1951,40 @@ model_deployments:
 
   - name: together/llama-3-70b-chat
     model_name: meta/llama-3-70b-chat
-    tokenizer_name: meta/llama-3-8b
-    max_sequence_length: 8191
+    tokenizer_name: meta/llama-3-8b-instruct
+    max_sequence_length: 8182
     client_spec:
       class_name: "helm.clients.together_client.TogetherChatClient"
       args:
         together_model: meta-llama/Llama-3-70b-chat-hf
 
+  - name: together/llama-3.2-3b-instruct-turbo
+    model_name: meta/llama-3.2-3b-instruct-turbo
+    tokenizer_name: meta/llama-3.2-3b-instruct
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        together_model: meta-llama/Llama-3.2-3B-Instruct-Turbo
+
+  - name: together/llama-3.2-11b-vision-instruct-turbo
+    model_name: meta/llama-3.2-11b-vision-instruct-turbo
+    tokenizer_name: meta/llama-3.2-11b-vision-instruct
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        together_model: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo
+
+  - name: together/llama-3.2-90b-vision-instruct-turbo
+    model_name: meta/llama-3.2-90b-vision-instruct-turbo
+    tokenizer_name: meta/llama-3.2-11b-vision-instruct
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        together_model: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo
+
   - name: together/llama-guard-7b
     model_name: meta/llama-guard-7b
     tokenizer_name: meta-llama/Llama-2-7b-hf
@@ -1903,6 +2003,15 @@ model_deployments:
       args:
         together_model: meta-llama/llamaguard-2-8b
 
+  - name: together/llama-guard-3-8b
+    model_name: meta/llama-guard-3-8b
+    tokenizer_name: meta/llama-3.1-8b
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-Guard-3-8B
+
   # 01.AI
   - name: together/yi-6b
     model_name: 01-ai/yi-6b
@@ -2029,7 +2138,7 @@ model_deployments:
   - name: together/mixtral-8x7b-instruct-v0.1
     model_name: mistralai/mixtral-8x7b-instruct-v0.1
     tokenizer_name: mistralai/Mistral-7B-v0.1
-    max_sequence_length: 4095 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    max_sequence_length: 32767
     client_spec:
       class_name: "helm.clients.together_client.TogetherChatClient"
 
@@ -2165,18 +2274,6 @@ model_deployments:
       args:
         together_model: togethercomputer/RedPajama-INCITE-7B-Instruct
 
-  ## Tsinghua
-  - name: together/glm
-    deprecated: true  # Removed from Together
-    model_name: tsinghua/glm
-    tokenizer_name: TsinghuaKEG/ice
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.clients.together_client.TogetherClient"
-    window_service_spec:
-      class_name: "helm.benchmark.window_services.ice_window_service.ICEWindowService"
-
   - name: thudm/cogview2
     model_name: thudm/cogview2
     tokenizer_name: openai/clip-vit-large-patch14
@@ -2215,22 +2312,6 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.palmyra_client.PalmyraClient"
 
-  - name: writer/palmyra-instruct-30
-    model_name: writer/palmyra-instruct-30
-    tokenizer_name: writer/gpt2
-    max_sequence_length: 2048
-    max_sequence_and_generated_tokens_length: 2048
-    client_spec:
-      class_name: "helm.clients.palmyra_client.PalmyraClient"
-
-  - name: writer/palmyra-e
-    model_name: writer/palmyra-e
-    tokenizer_name: writer/gpt2
-    max_sequence_length: 2048
-    max_sequence_and_generated_tokens_length: 2048
-    client_spec:
-      class_name: "helm.clients.palmyra_client.PalmyraClient"
-
   - name: writer/silk-road
     model_name: writer/silk-road
     tokenizer_name: writer/gpt2
@@ -2279,6 +2360,40 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.vision_language.palmyra_vision_client.PalmyraVisionClient"
 
+  - name: writer/palmyra-x-004
+    model_name: writer/palmyra-x-004
+    # Actual tokenizer is Llama 2, but it cannot be used in HELM due to this issue:
+    # https://github.com/stanford-crfm/helm/issues/2467
+    # Work around by using Llama 3 tokenizer for now.
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraChatClient"
+
+  - name: writer/palmyra-med-32k
+    model_name: writer/palmyra-med-32k
+    # Palmyra-Med uses the "<|end_of_text|>" as the end of text token, which is used by meta/llama-3-8b,
+    # rather than "<|eot_id|>", which is used by meta/llama-3-8b-instruct
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 32000
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraChatClient"
+
+  - name: writer/palmyra-med
+    model_name: writer/palmyra-med
+    # Palmyra-Med uses the "<|end_of_text|>" as the end of text token, which is used by meta/llama-3-8b,
+    # rather than "<|eot_id|>", which is used by meta/llama-3-8b-instruct
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraChatClient"
+
+  - name: writer/palmyra-fin-32k
+    model_name: writer/palmyra-fin-32k
+    tokenizer_name: meta/llama-3-8b-instruct
+    max_sequence_length: 32000
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraChatClient"
 
   # Qwen
 
@@ -2431,4 +2546,4 @@ model_deployments:
     tokenizer_name: openai/cl100k_base
     max_sequence_length: 64000
     client_spec:
-      class_name: "helm.clients.reka_client.RekaClient"
+      class_name: "helm.clients.reka_client.RekaClient"