crfm-helm 0.5.5__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/config/model_deployments.yaml

@@ -16,161 +16,6 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.simple_client.SimpleClient"
 
-  # Stanford Health Care
-  # Placed earlier in the file to make them non-default
-  - name: stanfordhealthcare/claude-3-5-sonnet-20241022
-    model_name: anthropic/claude-3-5-sonnet-20241022
-    tokenizer_name: anthropic/claude
-    max_sequence_length: 200000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
-      args:
-        model: anthropic.claude-3-5-sonnet-20241022-v2:0
-        deployment: Claude35Sonnetv2/awssig4fa
-  
-  - name: stanfordhealthcare/claude-3-7-sonnet-20250219
-    model_name: anthropic/claude-3-7-sonnet-20250219
-    tokenizer_name: anthropic/claude
-    max_sequence_length: 200000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
-      args:
-        model: arn:aws:bedrock:us-west-2:679683451337:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0
-        deployment: awssig4claude37/aswsig4claude37
-
-  - name: stanfordhealthcare/gemini-1.5-pro-001
-    model_name: google/gemini-1.5-pro-001
-    tokenizer_name: google/gemma-2b
-    max_sequence_length: 1000000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_google_client.StanfordHealthCareGoogleClient"
-      args:
-        deployment: gcpgemini/apim-gcp-oauth-fa
-
-  - name: stanfordhealthcare/gemini-2.0-flash-001
-    model_name: google/gemini-2.0-flash-001
-    tokenizer_name: google/gemma-2b
-    max_sequence_length: 1000000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_google_client.StanfordHealthCareGoogleClient"
-      args:
-        deployment: gcp-gem20flash-fa/apim-gcp-gem20flash-fa
-
-  - name: stanfordhealthcare/gpt-4o-mini-2024-07-18
-    model_name: openai/gpt-4o-mini-2024-07-18
-    tokenizer_name: openai/o200k_base
-    max_sequence_length: 128000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
-      args:
-        openai_model_name: gpt-4o-mini
-        api_version: 2023-05-15
-
-  - name: stanfordhealthcare/gpt-4o-2024-05-13
-    model_name: openai/gpt-4o-2024-05-13
-    tokenizer_name: openai/o200k_base
-    max_sequence_length: 128000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
-      args:
-        openai_model_name: gpt-4o
-        api_version: 2023-05-15
-  
-  - name: stanfordhealthcare/gpt-4-0613
-    model_name: openai/gpt-4-0613
-    tokenizer_name: openai/o200k_base
-    max_sequence_length: 8192
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
-      args:
-        openai_model_name: gpt-4
-        api_version: 2023-05-15
-
-  - name: stanfordhealthcare/gpt-4-turbo-2024-04-09
-    model_name: openai/gpt-4-turbo-2024-04-09
-    tokenizer_name: openai/cl100k_base
-    max_sequence_length: 128000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
-      args:
-        openai_model_name: gpt-4-turbo
-        api_version: 2023-05-15
-
-  - name: stanfordhealthcare/o3-mini-2025-01-31
-    model_name: openai/o3-mini-2025-01-31
-    tokenizer_name: openai/cl100k_base
-    max_sequence_length: 200000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
-      args:
-        openai_model_name: o3-mini
-        api_version: 2024-12-01-preview
-        base_url: "{endpoint}/openai-eastus2"
-
-  - name: stanfordhealthcare/o1-2024-12-17
-    model_name: openai/o1-2024-12-17
-    tokenizer_name: openai/cl100k_base
-    max_sequence_length: 128000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
-      args:
-        openai_model_name: o1
-        api_version: 2024-12-01-preview
-        base_url: "{endpoint}/openai-eastus2"
-
-  - name: stanfordhealthcare/deepseek-r1
-    model_name: deepseek-ai/deepseek-r1
-    tokenizer_name: deepseek-ai/deepseek-r1
-    max_sequence_length: 128000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
-      args:
-        openai_model_name: deepseek-chat
-        output_processor: helm.benchmark.metrics.output_processors.remove_deepseek_r1_thinking
-        base_url: "{endpoint}/deepseekr1/v1"
-
-  - name: stanfordhealthcare/llama-3.3-70b-instruct
-    model_name: meta/llama-3.3-70b-instruct
-    tokenizer_name: meta/llama-3.3-70b-instruct
-    max_sequence_length: 128000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
-      args:
-        base_url: "{endpoint}/llama3370b/v1"
-
-  - name: stanfordhealthcare/phi-3.5-mini-instruct
-    model_name: microsoft/phi-3.5-mini-instruct
-    tokenizer_name: microsoft/phi-3.5-mini-instruct
-    max_sequence_length: 131072
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
-      args:
-        base_url: "{endpoint}/phi35mi/v1"
-
-  - name: stanfordhealthcare_shc/gpt-4o-2024-05-13
-    model_name: openai/gpt-4o-2024-05-13
-    tokenizer_name: openai/o200k_base
-    max_sequence_length: 128000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_shc_openai_client.StanfordHealthCareSHCOpenAIClient"
-      deployment: gpt-4o
-
-  - name: stanfordhealthcare_shc/gpt-4o-mini-2024-07-18
-    model_name: openai/gpt-4o-mini-2024-07-18
-    tokenizer_name: openai/o200k_base
-    max_sequence_length: 128000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_shc_openai_client.StanfordHealthCareSHCOpenAIClient"
-      deployment: gpt-4o-mini
-  
-  - name: stanfordhealthcare_shc/gpt-4-turbo-2024-04-09
-    model_name: openai/gpt-4-turbo-2024-04-09
-    tokenizer_name: openai/cl100k_base
-    max_sequence_length: 128000
-    client_spec:
-      class_name: "helm.clients.stanfordhealthcare_shc_openai_client.StanfordHealthCareSHCOpenAIClient"
-      deployment: gpt-4-turbo-2024-04-09
-
   # Adobe
   - name: adobe/giga-gan
     model_name: adobe/giga-gan
@@ -260,6 +105,14 @@ model_deployments:
 
 
   # Amazon nova models
+  - name: amazon/nova-premier-v1:0
+    model_name: amazon/nova-premier-v1:0
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 1000000
+    client_spec:
+      class_name: "helm.clients.bedrock_client.BedrockNovaClient"
+      args:
+        bedrock_model_id: us.amazon.nova-premier-v1:0
 
   - name: amazon/nova-pro-v1:0
     model_name: amazon/nova-pro-v1:0
@@ -463,6 +316,53 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
 
+  - name: anthropic/claude-3-7-sonnet-20250219-thinking-10k
+    model_name: anthropic/claude-3-7-sonnet-20250219-thinking-10k
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
+      args:
+        anthropic_model_name: claude-3-7-sonnet-20250219
+        thinking_budget_tokens: 10000
+        stream: true
+
+  - name: anthropic/claude-sonnet-4-20250514
+    model_name: anthropic/claude-sonnet-4-20250514
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
+
+  - name: anthropic/claude-sonnet-4-20250514-thinking-10k
+    model_name: anthropic/claude-sonnet-4-20250514-thinking-10k
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
+      args:
+        anthropic_model_name: claude-sonnet-4-20250514
+        thinking_budget_tokens: 10000
+        stream: true
+
+  - name: anthropic/claude-opus-4-20250514
+    model_name: anthropic/claude-opus-4-20250514
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
+
+  - name: anthropic/claude-opus-4-20250514-thinking-10k
+    model_name: anthropic/claude-opus-4-20250514-thinking-10k
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
+      args:
+        anthropic_model_name: claude-opus-4-20250514
+        thinking_budget_tokens: 10000
+        stream: true
+
   - name: anthropic/stanford-online-all-v4-s3
     deprecated: true # Closed model, not accessible via API
     model_name: anthropic/stanford-online-all-v4-s3
@@ -583,25 +483,16 @@ model_deployments:
       args:
         disable_logprobs: True
 
-  - name: together/deepseek-r1
-    model_name: deepseek-ai/deepseek-r1
-    tokenizer_name: deepseek-ai/deepseek-r1
-    max_sequence_length: 32768
-    client_spec:
-      class_name: "helm.clients.together_client.TogetherChatClient"
-      args:
-        disable_logprobs: True
-
-  - name: together/deepseek-r1-hide-reasoning
-    model_name: deepseek-ai/deepseek-r1-hide-reasoning
+  - name: together/deepseek-r1-0528
+    model_name: deepseek-ai/deepseek-r1-0528
     tokenizer_name: deepseek-ai/deepseek-r1
     max_sequence_length: 32768
     client_spec:
       class_name: "helm.clients.together_client.TogetherChatClient"
       args:
         together_model: deepseek-ai/deepseek-r1
+        parse_thinking: true
         disable_logprobs: True
-        output_processor: helm.benchmark.metrics.output_processors.remove_deepseek_r1_thinking
 
   # Gooseai
 
@@ -802,6 +693,14 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.vertexai_client.VertexAIChatClient"
 
+  - name: google/gemini-2.0-flash-lite-001
+    model_name: google/gemini-2.0-flash-lite-001
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
+    # TODO: Max output tokens: 8192
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
   - name: google/gemini-2.0-flash-thinking-exp-01-21
     model_name: google/gemini-2.0-flash-thinking-exp-01-21
     tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
@@ -809,6 +708,75 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.vertexai_client.VertexAIChatClient"
 
+  - name: google/gemini-2.5-flash-lite-preview-06-17
+    model_name: google/gemini-2.5-flash-lite-preview-06-17
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1048576  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-flash
+    # TODO: Max output tokens: 65536
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+      args:
+        # Only the global location is supported. See:
+        # - https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-flash-lite
+        # - https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations#global-endpoint
+        location: global
+
+  - name: google/gemini-2.5-flash-preview-04-17
+    model_name: google/gemini-2.5-flash-preview-04-17
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1048576  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-flash
+    # TODO: Max output tokens: 65536
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-2.5-flash-preview-05-20
+    model_name: google/gemini-2.5-flash-preview-05-20
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1048576  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-flash
+    # TODO: Max output tokens: 65536
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-2.5-flash
+    model_name: google/gemini-2.5-flash
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1048576  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-flash
+    # TODO: Max output tokens: 65536
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-2.5-pro-exp-03-25
+    model_name: google/gemini-2.5-pro-exp-03-25
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1048576   # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-pro
+    # TODO: Max output tokens: 65536
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-2.5-pro-preview-03-25
+    model_name: google/gemini-2.5-pro-preview-03-25
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1048576   # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-pro
+    # TODO: Max output tokens: 65536
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-2.5-pro-preview-05-06
+    model_name: google/gemini-2.5-pro-preview-05-06
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1048576   # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-pro
+    # TODO: Max output tokens: 65536
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-2.5-pro
+    model_name: google/gemini-2.5-pro
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1048576   # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/2-5-pro
+    # TODO: Max output tokens: 65536
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
   - name: google/gemini-1.5-flash-8b-001
     model_name: google/gemini-1.5-flash-8b-001
     tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
@@ -1364,6 +1332,8 @@ model_deployments:
     max_sequence_length: 4096
     client_spec:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        apply_chat_template: false
     
   ## KAIST AI
   - name: huggingface/prometheus-vision-13b-v1.0-hf
@@ -1385,6 +1355,23 @@ model_deployments:
         tokenizer_name: "anas-awadalla-2/mpt-7b"
         cross_attn_every_n_layers: 4
 
+  ## Marin Community
+  - name: huggingface/marin-8b-instruct
+    model_name: marin-community/marin-8b-instruct
+    tokenizer_name: marin-community/marin-8b-instruct
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  - name: together/marin-8b-instruct
+    model_name: marin-community/marin-8b-instruct
+    tokenizer_name: marin-community/marin-8b-instruct
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient" 
+
   ## Microsoft
   - name: together/phi-2
     model_name: microsoft/phi-2
@@ -1530,6 +1517,8 @@ model_deployments:
     max_sequence_length: 32768
     client_spec:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        apply_chat_template: false
 
   - name: huggingface/sailor-7b-chat
     model_name: sail/sailor-7b-chat
@@ -1546,6 +1535,7 @@ model_deployments:
       class_name: "helm.clients.huggingface_client.HuggingFaceClient"
       args:
         device_map: auto
+        apply_chat_template: false
 
   - name: huggingface/sailor-14b-chat
     model_name: sail/sailor-14b-chat
@@ -1980,6 +1970,13 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.mistral_client.MistralAIClient"
 
+  - name: mistralai/mistral-medium-2505
+    model_name: mistralai/mistral-medium-2505
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.mistral_client.MistralAIClient"
+
   - name: mistralai/mistral-large-2402
     model_name: mistralai/mistral-large-2402
     tokenizer_name: mistralai/Mistral-7B-v0.1
@@ -2223,12 +2220,47 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.openai_client.OpenAIClient"
 
-  - name: openai/whisper-1_gpt-4o-2024-11-20
-    model_name: openai/whisper-1_gpt-4o-2024-11-20
+  - name: openai/gpt-4.1-2025-04-14
+    model_name: openai/gpt-4.1-2025-04-14
     tokenizer_name: openai/o200k_base
-    max_sequence_length: 128000
+    max_sequence_length: 1047576
     client_spec:
-      class_name: "helm.clients.openai_client.OpenAITranscriptionThenCompletionClient"
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4.1-mini-2025-04-14
+    model_name: openai/gpt-4.1-mini-2025-04-14
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 1047576
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4.1-nano-2025-04-14
+    model_name: openai/gpt-4.1-nano-2025-04-14
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 1047576
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/whisper-1_gpt-4o-2024-11-20
+    model_name: openai/whisper-1_gpt-4o-2024-11-20
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAITranscriptionThenCompletionClient"
+
+  - name: openai/gpt-4o-transcribe_gpt-4o-2024-11-20
+    model_name: openai/gpt-4o-transcribe_gpt-4o-2024-11-20
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAITranscriptionThenCompletionClient"
+
+  - name: openai/gpt-4o-mini-transcribe_gpt-4o-2024-11-20
+    model_name: openai/gpt-4o-mini-transcribe_gpt-4o-2024-11-20
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAITranscriptionThenCompletionClient"
 
   - name: openai/gpt-4o-audio-preview-2024-10-01
     model_name: openai/gpt-4o-audio-preview-2024-10-01
@@ -2278,6 +2310,33 @@ model_deployments:
       class_name: "helm.clients.openai_client.OpenAIClient"
 
   ## o1 Models
+  - name: openai/o1-pro-2025-03-19
+    model_name: openai/o1-pro-2025-03-19
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_responses_client.OpenAIResponseClient"
+
+  - name: openai/o1-pro-2025-03-19-low-reasoning-effort
+    model_name: openai/o1-pro-2025-03-19-low-reasoning-effort
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_responses_client.OpenAIResponseClient"
+      args:
+        openai_model_name: o1-pro-2025-03-19
+        reasoning_effort: low
+
+  - name: openai/o1-pro-2025-03-19-high-reasoning-effort
+    model_name: openai/o1-pro-2025-03-19-high-reasoning-effort
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_responses_client.OpenAIResponseClient"
+      args:
+        openai_model_name: o1-pro-2025-03-19
+        reasoning_effort: high
+
   - name: openai/o1-2024-12-17
     model_name: openai/o1-2024-12-17
     tokenizer_name: openai/cl100k_base
@@ -2346,6 +2405,73 @@ model_deployments:
         openai_model_name: o3-mini-2025-01-31
         reasoning_effort: high
 
+  - name: openai/o3-2025-04-16
+    model_name: openai/o3-2025-04-16
+    tokenizer_name: openai/cl100k_base
+    # Source: https://platform.openai.com/docs/models/o3
+    max_sequence_length: 200000
+    # TODO: max_output_tokens: 100000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/o3-2025-04-16-low-reasoning-effort
+    model_name: openai/o3-2025-04-16-low-reasoning-effort
+    tokenizer_name: openai/cl100k_base
+    # Source: https://platform.openai.com/docs/models/o3
+    max_sequence_length: 200000
+    # TODO: max_output_tokens: 100000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+      args:
+        openai_model_name: o3-2025-04-16
+        reasoning_effort: low
+
+  - name: openai/o3-2025-04-16-high-reasoning-effort
+    model_name: openai/o3-2025-04-16-high-reasoning-effort
+    tokenizer_name: openai/cl100k_base
+    # Source: https://platform.openai.com/docs/models/o3
+    max_sequence_length: 200000
+    # TODO: max_output_tokens: 100000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+      args:
+        openai_model_name: o3-2025-04-16
+        reasoning_effort: high
+
+  - name: openai/o4-mini-2025-04-16
+    model_name: openai/o4-mini-2025-04-16
+    tokenizer_name: openai/cl100k_base
+    # Source: https://platform.openai.com/docs/models/o4-mini
+    max_sequence_length: 200000
+    # TODO: max_output_tokens: 100000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/o4-mini-2025-04-16-low-reasoning-effort
+    model_name: openai/o4-mini-2025-04-16-low-reasoning-effort
+    tokenizer_name: openai/cl100k_base
+    # Source: https://platform.openai.com/docs/models/o4-mini
+    max_sequence_length: 200000
+    # TODO: max_output_tokens: 100000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+      args:
+        openai_model_name: o4-mini-2025-04-16
+        reasoning_effort: low
+
+
+  - name: openai/o4-mini-2025-04-16-high-reasoning-effort
+    model_name: openai/o4-mini-2025-04-16-high-reasoning-effort
+    tokenizer_name: openai/cl100k_base
+    # Source: https://platform.openai.com/docs/models/o4-mini
+    max_sequence_length: 200000
+    # TODO: max_output_tokens: 100000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+      args:
+        openai_model_name: o4-mini-2025-04-16
+        reasoning_effort: high
+
   ## Text Similarity Models
   # OpenAI similarity embedding models: https://beta.openai.com/docs/guides/embeddings
   # The number of parameters is guessed based on the number of parameters of the
@@ -2610,6 +2736,24 @@ model_deployments:
       args:
         together_model: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo
 
+  - name: together/llama-4-scout-17b-16e-instruct
+    model_name: meta/llama-4-scout-17b-16e-instruct
+    tokenizer_name: meta/llama-4-scout-17b-16e-instruct
+    max_sequence_length: 327680
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        together_model: meta-llama/Llama-4-Scout-17B-16E-Instruct
+
+  - name: together/llama-4-maverick-17b-128e-instruct-fp8
+    model_name: meta/llama-4-maverick-17b-128e-instruct-fp8
+    tokenizer_name: meta/llama-4-scout-17b-16e-instruct
+    max_sequence_length: 524288
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        together_model: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8
+
   - name: together/llama-3-8b-chat
     model_name: meta/llama-3-8b-chat
     tokenizer_name: meta/llama-3-8b-instruct
@@ -2784,6 +2928,42 @@ model_deployments:
       args:
         pretrained_model_name_or_path: allenai/OLMo-1.7-7B-hf
 
+  - name: huggingface/olmo-2-1124-7b-instruct
+    model_name: allenai/olmo-2-1124-7b-instruct
+    tokenizer_name: allenai/olmo-2-1124-7b-instruct
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  - name: huggingface/olmo-2-1124-13b-instruct
+    model_name: allenai/olmo-2-1124-13b-instruct
+    tokenizer_name: allenai/olmo-2-1124-7b-instruct
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  - name: huggingface/olmo-2-0325-32b-instruct
+    model_name: allenai/olmo-2-0325-32b-instruct
+    tokenizer_name: allenai/olmo-2-0325-32b-instruct
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
+  - name: huggingface/olmoe-1b-7b-0125-instruct
+    model_name: allenai/olmoe-1b-7b-0125-instruct
+    tokenizer_name: allenai/olmoe-1b-7b-0125-instruct
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        device_map: auto
+
   ## MistralAI
   - name: together/mistral-7b-v0.1
     model_name: mistralai/mistral-7b-v0.1
@@ -3069,6 +3249,14 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.palmyra_client.PalmyraChatClient"
 
+  - name: writer/palmyra-x5
+    model_name: writer/palmyra-x5
+    # See tokenizer comment for writer/palmyra-x-004
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 1000000
+    client_spec:
+      class_name: "helm.clients.writer_client.WriterClient"
+
   - name: writer/palmyra-med-32k
     model_name: writer/palmyra-med-32k
     # Palmyra-Med uses the "<|end_of_text|>" as the end of text token, which is used by meta/llama-3-8b,
@@ -3080,12 +3268,10 @@ model_deployments:
 
   - name: writer/palmyra-med
     model_name: writer/palmyra-med
-    # Palmyra-Med uses the "<|end_of_text|>" as the end of text token, which is used by meta/llama-3-8b,
-    # rather than "<|eot_id|>", which is used by meta/llama-3-8b-instruct
     tokenizer_name: meta/llama-3-8b
-    max_sequence_length: 4096
+    max_sequence_length: 32000
     client_spec:
-      class_name: "helm.clients.palmyra_client.PalmyraChatClient"
+      class_name: "helm.clients.writer_client.WriterClient"
 
   - name: writer/palmyra-fin-32k
     model_name: writer/palmyra-fin-32k
@@ -3104,16 +3290,23 @@ model_deployments:
 
   # xAI
 
-  - name: xai/grok-beta
-    model_name: xai/grok-beta
-    # No public information on tokenizer, so just pick an arbitrary one.
-    # It shouldn't matter since the context is long.
-    tokenizer_name: openai/o200k_base
-    max_sequence_length: 128000
+  - name: xai/grok-3-beta
+    model_name: xai/grok-3-beta
+    tokenizer_name: xai/grok-3-beta
+    max_sequence_length: 131072
     client_spec:
-      class_name: "helm.clients.openai_client.OpenAIClient"
-      args:
-        base_url: https://api.x.ai/v1
+      class_name: "helm.clients.grok_client.GrokChatClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
+
+  - name: xai/grok-3-mini-beta
+    model_name: xai/grok-3-mini-beta
+    tokenizer_name: xai/grok-3-mini-beta
+    max_sequence_length: 131072
+    client_spec:
+      class_name: "helm.clients.grok_client.GrokChatClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
 
   # Qwen
 
@@ -3217,6 +3410,15 @@ model_deployments:
     max_sequence_length: 128000
     client_spec:
       class_name: "helm.clients.together_client.TogetherChatClient"
+
+  - name: together/qwen3-235b-a22b-fp8-tput
+    model_name: qwen/qwen3-235b-a22b-fp8-tput
+    tokenizer_name: qwen/qwen3-235b-a22b
+    max_sequence_length: 40960
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+      args:
+        parse_thinking: true
   
   - name: huggingface/qwen2.5-7b-instruct-4bit
     model_name: qwen/qwen2.5-7b-instruct
@@ -3240,6 +3442,60 @@ model_deployments:
       args:
         pretrained_model_name_or_path: Qwen/Qwen2.5-7B-Instruct
 
+  - name: huggingface/smollm2-135m
+    model_name: huggingface/smollm2-135m
+    tokenizer_name: huggingface/smollm2-135m
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_pipeline_client.HuggingFacePipelineClient"
+      args:
+        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-135M
+
+  - name: huggingface/smollm2-360m
+    model_name: huggingface/smollm2-360m
+    tokenizer_name: huggingface/smollm2-135m
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_pipeline_client.HuggingFacePipelineClient"
+      args:
+        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-360M
+
+  - name: huggingface/smollm2-1.7b
+    model_name: huggingface/smollm2-1.7b
+    tokenizer_name: huggingface/smollm2-135m
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_pipeline_client.HuggingFacePipelineClient"
+      args:
+        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-1.7B
+
+  - name: huggingface/smollm2-135m-instruct
+    model_name: huggingface/smollm2-135m-instruct
+    tokenizer_name: huggingface/smollm2-135m-instruct
+    max_sequence_length: 328192768
+    client_spec:
+      class_name: "helm.clients.huggingface_pipeline_client.HuggingFacePipelineClient"
+      args:
+        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-135M-Instruct
+
+  - name: huggingface/smollm2-360m-instruct
+    model_name: huggingface/smollm2-360m-instruct
+    tokenizer_name: huggingface/smollm2-135m-instruct
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_pipeline_client.HuggingFacePipelineClient"
+      args:
+        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-360M-Instruct
+
+  - name: huggingface/smollm2-1.7b-instruct
+    model_name: huggingface/smollm2-1.7b-instruct
+    tokenizer_name: huggingface/smollm2-135m-instruct
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_pipeline_client.HuggingFacePipelineClient"
+      args:
+        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-1.7B-Instruct
+
   - name: together/qwq-32b-preview
     model_name: qwen/qwq-32b-preview
     tokenizer_name: qwen/qwq-32b-preview
@@ -3275,6 +3531,34 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.vision_language.qwen2_vlm_client.Qwen2VLMClient"
 
+  - name: huggingface/qwen2.5-vl-3b-instruct
+    model_name: qwen/qwen2.5-vl-3b-instruct
+    tokenizer_name: qwen/qwen-vl-chat
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.vision_language.qwen2_vlm_client.Qwen2VLMClient"
+
+  - name: huggingface/qwen2.5-vl-7b-instruct
+    model_name: qwen/qwen2.5-vl-7b-instruct
+    tokenizer_name: qwen/qwen-vl-chat
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.vision_language.qwen2_vlm_client.Qwen2VLMClient"
+
+  - name: huggingface/qwen2.5-vl-32b-instruct
+    model_name: qwen/qwen2.5-vl-32b-instruct
+    tokenizer_name: qwen/qwen-vl-chat
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.vision_language.qwen2_vlm_client.Qwen2VLMClient"
+
+  - name: huggingface/qwen2.5-vl-72b-instruct
+    model_name: qwen/qwen2.5-vl-72b-instruct
+    tokenizer_name: qwen/qwen-vl-chat
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.vision_language.qwen2_vlm_client.Qwen2VLMClient"
+
   - name: huggingface/qwen-audio-chat
     model_name: qwen/qwen-audio-chat
     tokenizer_name: qwen/qwen-audio-chat
@@ -3289,6 +3573,13 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.audio_language.qwen2_audiolm_client.Qwen2AudioLMClient"
 
+  - name: huggingface/qwen2.5-omni-7b
+    model_name: qwen/qwen2.5-omni-7b
+    tokenizer_name: qwen/qwen2.5-omni-7b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.audio_language.qwen2_5_omni_client.Qwen2_5OmniAudioLMClient"
+
 # Reka
   - name: reka/reka-core
     model_name: reka/reka-core
@@ -3567,8 +3858,6 @@ model_deployments:
         watsonx_model_name: ibm/granite-3-2b-instruct
         region: Dallas
 
-
-#
   - name: ibm/granite-3-8b-instruct
     model_name: ibm/granite-3.1-8b-instruct
     tokenizer_name: ibm-granite/granite-3.1-8b-instruct
@@ -3578,9 +3867,7 @@ model_deployments:
       args:
         watsonx_model_name: ibm/granite-3-8b-instruct
         region: Dallas
-#
 
-#
   - name: ibm/granite-13b-instruct-v2
     model_name: ibm/granite-13b-instruct-v2
     tokenizer_name: EleutherAI/gpt-neox-20b
@@ -3590,7 +3877,7 @@ model_deployments:
       args:
         watsonx_model_name: ibm/granite-13b-instruct-v2
         region: Dallas
-#
+
   - name: ibm/granite-20b-code-instruct-8k
     model_name: ibm/granite-20b-code-instruct-8k
     tokenizer_name: ibm-granite/granite-20b-code-instruct-8k
@@ -3600,7 +3887,7 @@ model_deployments:
       args:
         watsonx_model_name: ibm/granite-20b-code-instruct
         region: Dallas
-#
+
   - name: ibm/granite-34b-code-instruct
     model_name: ibm/granite-34b-code-instruct
     tokenizer_name: ibm-granite/granite-34b-code-instruct-8k
@@ -3610,7 +3897,7 @@ model_deployments:
       args:
         watsonx_model_name: ibm/granite-34b-code-instruct
         region: Dallas
-#
+
   - name: ibm/granite-3b-code-instruct
     model_name: ibm/granite-3b-code-instruct
     tokenizer_name: ibm-granite/granite-3b-code-instruct-128k
@@ -3620,7 +3907,7 @@ model_deployments:
       args:
         watsonx_model_name: ibm/granite-3b-code-instruct
         region: Dallas
-#
+
   - name: ibm/granite-8b-code-instruct
     model_name: ibm/granite-8b-code-instruct
     tokenizer_name: ibm-granite/granite-8b-code-instruct-128k
@@ -3639,4 +3926,388 @@ model_deployments:
       class_name: "helm.clients.ibm_client.IbmChatClient"
       args:
         watsonx_model_name: mistralai/mixtral-8x7b-instruct-v01
-        region: Dallas
+        region: Dallas
+
+  - name: ibm/granite-3.3-8b-instruct
+    model_name: ibm/granite-3.3-8b-instruct
+    tokenizer_name: ibm/granite-3.3-8b-instruct
+    max_sequence_length: 131072
+    client_spec:
+      class_name: "helm.clients.ibm_client.IbmTextClient"
+      args:
+        watsonx_model_name: ibm/granite-3-3-8b-instruct
+        region: Dallas
+
+  # Vietnamese
+  - name: ura-hcmut/ura-llama-2.1-8b
+    model_name: ura-hcmut/ura-llama-2.1-8b
+    tokenizer_name: meta/llama-3.1-8b-instruct
+    max_sequence_length: 131072
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: ura-hcmut/ura-llama-2.1-8b
+
+  - name: ura-hcmut/ura-llama-2-8b
+    model_name: ura-hcmut/ura-llama-2-8b
+    tokenizer_name: meta/llama-3-8b-instruct
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: ura-hcmut/ura-llama-2-8b
+
+  - name: ura-hcmut/ura-llama-7b
+    model_name: ura-hcmut/ura-llama-7b
+    tokenizer_name: meta-llama/Llama-2-7b-hf
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: ura-hcmut/ura-llama-7b
+
+  - name: ura-hcmut/ura-llama-13b
+    model_name: ura-hcmut/ura-llama-13b
+    tokenizer_name: meta-llama/Llama-2-7b-hf
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: ura-hcmut/ura-llama-13b
+
+  - name: ura-hcmut/ura-llama-70b
+    model_name: ura-hcmut/ura-llama-70b
+    tokenizer_name: meta-llama/Llama-2-7b-hf
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: ura-hcmut/ura-llama-70b
+
+  - name: ura-hcmut/GemSUra-7B
+    model_name: ura-hcmut/GemSUra-7B
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: ura-hcmut/GemSUra-7B
+
+  - name: ura-hcmut/GemSUra-2B
+    model_name: ura-hcmut/GemSUra-2B
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: ura-hcmut/GemSUra-2B
+
+  - name: ura-hcmut/MixSUra
+    model_name: ura-hcmut/MixSUra
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: ura-hcmut/MixSUra
+
+  - name: vilm/vinallama-7b-chat
+    model_name: vilm/vinallama-7b-chat
+    tokenizer_name: vilm/vinallama-7b-chat
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: vilm/vinallama-7b-chat
+
+  - name: vilm/vinallama-2.7b-chat
+    model_name: vilm/vinallama-2.7b-chat
+    tokenizer_name: vilm/vinallama-2.7b-chat
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: vilm/vinallama-2.7b-chat
+
+  - name: vilm/vietcuna-7b-v3
+    model_name: vilm/vietcuna-7b-v3
+    tokenizer_name: vilm/vietcuna-7b-v3
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: vilm/vietcuna-7b-v3
+
+  - name: vilm/vietcuna-3b-v2
+    model_name: vilm/vietcuna-3b-v2
+    tokenizer_name: vilm/vietcuna-7b-v3
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: vilm/vietcuna-3b-v2
+
+  - name: vilm/Quyen-v0.1
+    model_name: vilm/Quyen-v0.1
+    tokenizer_name: qwen/qwen2-72b-instruct
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: vilm/Quyen-v0.1
+
+  - name: vilm/Quyen-Plus-v0.1
+    model_name: vilm/Quyen-Plus-v0.1
+    tokenizer_name: qwen/qwen2-72b-instruct
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: vilm/Quyen-Plus-v0.1
+
+  - name: vilm/Quyen-Pro-v0.1
+    model_name: vilm/Quyen-Pro-v0.1
+    tokenizer_name: qwen/qwen2-72b-instruct
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: vilm/Quyen-Pro-v0.1
+  
+  - name: vilm/Quyen-Pro-Max-v0.1
+    model_name: vilm/Quyen-Pro-Max-v0.1
+    tokenizer_name: qwen/qwen2-72b-instruct
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: vilm/Quyen-Pro-Max-v0.1
+
+  - name: vilm/Quyen-Mini-v0.1
+    model_name: vilm/Quyen-Mini-v0.1
+    tokenizer_name: qwen/qwen2-72b-instruct
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: vilm/Quyen-Mini-v0.1
+
+  - name: vilm/Quyen-SE-v0.1
+    model_name: vilm/Quyen-SE-v0.1
+    tokenizer_name: qwen/qwen2-72b-instruct
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: vilm/Quyen-SE-v0.1
+
+  - name: Viet-Mistral/Vistral-7B-Chat
+    model_name: Viet-Mistral/Vistral-7B-Chat
+    tokenizer_name: Viet-Mistral/Vistral-7B-Chat
+    max_sequence_length: 32768
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: Viet-Mistral/Vistral-7B-Chat
+
+  - name: vinai/PhoGPT-7B5-Instruct
+    model_name: vinai/PhoGPT-7B5-Instruct
+    tokenizer_name: vinai/PhoGPT-7B5-Instruct
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: vinai/PhoGPT-7B5-Instruct
+
+  - name: vinai/PhoGPT-4B-Chat
+    model_name: vinai/PhoGPT-4B-Chat
+    tokenizer_name: vinai/PhoGPT-4B-Chat
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: vinai/PhoGPT-4B-Chat
+
+  # Stanford Health Care
+  # Placed later in the file to make them non-default
+  - name: stanfordhealthcare/claude-3-5-sonnet-20241022
+    model_name: anthropic/claude-3-5-sonnet-20241022
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
+      args:
+        model: anthropic.claude-3-5-sonnet-20241022-v2:0
+        deployment: Claude35Sonnetv2/awssig4fa
+  
+  - name: stanfordhealthcare/claude-3-7-sonnet-20250219
+    model_name: anthropic/claude-3-7-sonnet-20250219
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_claude_client.StanfordHealthCareClaudeClient"
+      args:
+        model: arn:aws:bedrock:us-west-2:679683451337:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0
+        deployment: awssig4claude37/aswsig4claude37
+
+  - name: stanfordhealthcare/gemini-1.5-pro-001
+    model_name: google/gemini-1.5-pro-001
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 1000000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_google_client.StanfordHealthCareGoogleClient"
+      args:
+        deployment: gcpgemini/apim-gcp-oauth-fa
+
+  - name: stanfordhealthcare/gemini-2.0-flash-001
+    model_name: google/gemini-2.0-flash-001
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 1000000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_google_client.StanfordHealthCareGoogleClient"
+      args:
+        deployment: gcp-gem20flash-fa/apim-gcp-gem20flash-fa
+
+  - name: stanfordhealthcare/gpt-4o-mini-2024-07-18
+    model_name: openai/gpt-4o-mini-2024-07-18
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
+      args:
+        openai_model_name: gpt-4o-mini
+        api_version: 2023-05-15
+
+  - name: stanfordhealthcare/gpt-4o-2024-05-13
+    model_name: openai/gpt-4o-2024-05-13
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
+      args:
+        openai_model_name: gpt-4o
+        api_version: 2023-05-15
+  
+  - name: stanfordhealthcare/gpt-4-0613
+    model_name: openai/gpt-4-0613
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
+      args:
+        openai_model_name: gpt-4
+        api_version: 2023-05-15
+
+  - name: stanfordhealthcare/gpt-4-turbo-2024-04-09
+    model_name: openai/gpt-4-turbo-2024-04-09
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
+      args:
+        openai_model_name: gpt-4-turbo
+        api_version: 2023-05-15
+
+  - name: stanfordhealthcare/gpt-4.1-2025-04-14
+    model_name: openai/gpt-4.1-2025-04-14
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 1047576
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
+      args:
+        openai_model_name: gpt-4.1
+        api_version: 2025-01-01-preview
+        base_url: "{endpoint}/openai-eastus2"
+
+  - name: stanfordhealthcare/o3-mini-2025-01-31
+    model_name: openai/o3-mini-2025-01-31
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
+      args:
+        openai_model_name: o3-mini
+        api_version: 2024-12-01-preview
+        base_url: "{endpoint}/openai-eastus2"
+
+  - name: stanfordhealthcare/o1-2024-12-17
+    model_name: openai/o1-2024-12-17
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_azure_openai_client.StanfordHealthCareAzureOpenAIClient"
+      args:
+        openai_model_name: o1
+        api_version: 2024-12-01-preview
+        base_url: "{endpoint}/openai-eastus2"
+
+  - name: stanfordhealthcare/deepseek-r1
+    model_name: deepseek-ai/deepseek-r1
+    tokenizer_name: deepseek-ai/deepseek-r1
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
+      args:
+        openai_model_name: deepseek-chat
+        output_processor: helm.benchmark.metrics.output_processors.remove_deepseek_r1_thinking
+        base_url: "{endpoint}/deepseekr1/v1"
+
+  - name: stanfordhealthcare/llama-3.3-70b-instruct
+    model_name: meta/llama-3.3-70b-instruct
+    tokenizer_name: meta/llama-3.3-70b-instruct
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
+      args:
+        base_url: "{endpoint}/llama3370b/v1"
+
+  - name: stanfordhealthcare/llama-4-scout-17b-16e-instruct
+    model_name: meta/llama-4-scout-17b-16e-instruct
+    tokenizer_name: meta/llama-4-scout-17b-16e-instruct
+    max_sequence_length: 327680
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
+      args:
+        base_url: "{endpoint}/llama4-scout/v1"
+  
+  - name: stanfordhealthcare/llama-4-maverick-17b-128e-instruct-fp8
+    model_name: meta/llama-4-maverick-17b-128e-instruct-fp8
+    tokenizer_name: meta/llama-4-scout-17b-16e-instruct
+    max_sequence_length: 524288
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
+      args:
+        base_url: "{endpoint}/llama4-maverick/v1"
+
+  - name: stanfordhealthcare/phi-3.5-mini-instruct
+    model_name: microsoft/phi-3.5-mini-instruct
+    tokenizer_name: microsoft/phi-3.5-mini-instruct
+    max_sequence_length: 131072
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_openai_client.StanfordHealthCareOpenAIClient"
+      args:
+        base_url: "{endpoint}/phi35mi/v1"
+
+  - name: stanfordhealthcare_shc/gpt-4o-2024-05-13
+    model_name: openai/gpt-4o-2024-05-13
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_shc_openai_client.StanfordHealthCareSHCOpenAIClient"
+      deployment: gpt-4o
+
+  - name: stanfordhealthcare_shc/gpt-4o-mini-2024-07-18
+    model_name: openai/gpt-4o-mini-2024-07-18
+    tokenizer_name: openai/o200k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_shc_openai_client.StanfordHealthCareSHCOpenAIClient"
+      deployment: gpt-4o-mini
+  
+  - name: stanfordhealthcare_shc/gpt-4-turbo-2024-04-09
+    model_name: openai/gpt-4-turbo-2024-04-09
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.stanfordhealthcare_shc_openai_client.StanfordHealthCareSHCOpenAIClient"
+      deployment: gpt-4-turbo-2024-04-09