crfm-helm 0.5.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/common/images_utils.py

@@ -1,8 +1,9 @@
 import base64
 import io
+
 import requests
 import shutil
-from typing import List, Optional
+from typing import List, Optional, Tuple
 from urllib.request import urlopen
 
 import numpy as np
@@ -28,6 +29,12 @@ def open_image(image_location: str) -> Image.Image:
     return image.convert("RGB")
 
 
+def get_dimensions(image_location: str) -> Tuple[int, int]:
+    """Returns the dimensions of the image."""
+    image: Image.Image = open_image(image_location)
+    return image.size
+
+
 def encode_base64(image_location: str, format="JPEG") -> str:
     """Returns the base64 representation of an image file."""
     image_file = io.BytesIO()
@@ -36,7 +43,7 @@ def encode_base64(image_location: str, format="JPEG") -> str:
     return base64.b64encode(image_file.getvalue()).decode("ascii")
 
 
-def copy_image(src: str, dest: str, width: Optional[int] = None, height: Optional[int] = None):
+def copy_image(src: str, dest: str, width: Optional[int] = None, height: Optional[int] = None) -> None:
     """
     Copies the image file from `src` path to `dest` path. If dimensions `width` and `height`
     are specified, resizes the image before copying. `src` can be a URL.
@@ -44,7 +51,7 @@ def copy_image(src: str, dest: str, width: Optional[int] = None, height: Optiona
     if (width is not None and height is not None) or is_url(src):
         image = open_image(src)
         if width is not None and height is not None:
-            image = image.resize((width, height), Image.ANTIALIAS)
+            image = image.resize((width, height), Image.Resampling.LANCZOS)
         image.save(dest)
     else:
         shutil.copy(src, dest)

helm/config/model_deployments.yaml

@@ -436,7 +436,7 @@ model_deployments:
 
   - name: google/gemini-pro-vision
     model_name: google/gemini-pro-vision
-    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    tokenizer_name: openai/cl100k_base
     max_sequence_length: 12288
     max_sequence_and_generated_tokens_length: 16384 # Officially max_sequence_length + 4096, in practice max_output_tokens <= 2048 for vision models
     client_spec:
@@ -709,7 +709,35 @@ model_deployments:
     max_sequence_length: 2048
     client_spec:
       class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
-      
+
+  - name: huggingface/llava-v1.6-vicuna-7b-hf
+    model_name: uw-madison/llava-v1.6-vicuna-7b-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+
+  - name: huggingface/llava-v1.6-vicuna-13b-hf
+    model_name: uw-madison/llava-v1.6-vicuna-13b-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+
+  - name: huggingface/llava-v1.6-mistral-7b-hf
+    model_name: uw-madison/llava-v1.6-mistral-7b-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+
+  - name: huggingface/llava-v1.6-34b-hf
+    model_name: uw-madison/llava-v1.6-34b-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+
   ## OpenFlamingo
   - name: openflamingo/OpenFlamingo-9B-vitl-mpt7b
     model_name: openflamingo/OpenFlamingo-9B-vitl-mpt7b
@@ -963,6 +991,15 @@ model_deployments:
       class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
 
   # HuggingFaceM4
+  - name: HuggingFaceM4/idefics2-8b
+    model_name: HuggingFaceM4/idefics2-8b
+    # From https://huggingface.co/docs/transformers/main/en/model_doc/idefics2,
+    # "constructs a IDEFICS2 processor which wraps a LLama tokenizer."
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vision2seq_client.HuggingFaceVision2SeqClient"
+
   - name: HuggingFaceM4/idefics-9b
     model_name: HuggingFaceM4/idefics-9b
     tokenizer_name: HuggingFaceM4/idefics-9b
@@ -1320,6 +1357,15 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.openai_client.OpenAIClient"
 
+  - name: openai/gpt-4-1106-vision-preview
+    model_name: openai/gpt-4-1106-vision-preview
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000  # According to https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo
+    max_request_length: 128001
+    max_sequence_and_generated_tokens_length: 132096
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
   ## Codex Models
   # DEPRECATED: Codex models have been shut down on March 23 2023.
 
@@ -1589,6 +1635,24 @@ model_deployments:
       args:
         together_model: meta-llama/Meta-Llama-3-70B
 
+  - name: together/llama-3-8b-chat
+    model_name: meta/llama-3-8b-chat
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3-8B
+
+  - name: together/llama-3-70b-chat
+    model_name: meta/llama-3-70b-chat
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3-70B
+
   # 01.AI
   - name: together/yi-6b
     model_name: 01-ai/yi-6b
@@ -1608,6 +1672,24 @@ model_deployments:
       args:
         together_model: zero-one-ai/Yi-34B
 
+  - name: together/yi-6b-chat
+    model_name: 01-ai/yi-6b-chat
+    tokenizer_name: 01-ai/Yi-6B
+    max_sequence_length: 4095
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: zero-one-ai/Yi-6B
+
+  - name: together/yi-34b-chat
+    model_name: 01-ai/yi-34b-chat
+    tokenizer_name: 01-ai/Yi-6B
+    max_sequence_length: 4095
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: zero-one-ai/Yi-34B
+
 
   # Allen Institute for AI
   - name: together/olmo-7b
@@ -1665,6 +1747,22 @@ model_deployments:
     client_spec:
       class_name: "helm.clients.together_client.TogetherClient"
 
+  - name: together/mixtral-8x22b-instruct-v0.1
+    model_name: mistralai/mixtral-8x22b-instruct-v0.1
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 65535
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+
+  ## Snowflake
+  - name: together/snowflake-arctic-instruct
+    model_name: snowflake/snowflake-arctic-instruct
+    tokenizer_name: snowflake/snowflake-arctic-instruct
+    max_sequence_length: 4000  # Lower than 4096 because of chat tokens
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherChatClient"
+
   ## Stanford
   - name: together/alpaca-7b
     model_name: stanford/alpaca-7b

helm/config/model_metadata.yaml

@@ -189,7 +189,7 @@ models:
 
   # Anthropic
   - name: anthropic/claude-v1.3
-    display_name: Anthropic Claude v1.3
+    display_name: Claude v1.3
     description: A 52B parameter language model, trained using reinforcement learning from human feedback [paper](https://arxiv.org/pdf/2204.05862.pdf).
     creator_organization_name: Anthropic
     access: limited
@@ -198,7 +198,7 @@ models:
     tags: [ANTHROPIC_CLAUDE_1_MODEL_TAG, TEXT_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, ABLATION_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
  
   - name: anthropic/claude-instant-v1
-    display_name: Anthropic Claude Instant V1
+    display_name: Claude Instant V1
     description: A lightweight version of Claude, a model trained using reinforcement learning from human feedback ([docs](https://www.anthropic.com/index/introducing-claude)).
     creator_organization_name: Anthropic
     access: limited
@@ -206,7 +206,7 @@ models:
     tags: [ANTHROPIC_CLAUDE_1_MODEL_TAG, TEXT_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, ABLATION_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
 
   - name: anthropic/claude-instant-1.2
-    display_name: Anthropic Claude Instant 1.2
+    display_name: Claude Instant 1.2
     description: A lightweight version of Claude, a model trained using reinforcement learning from human feedback ([docs](https://www.anthropic.com/index/introducing-claude)).
     creator_organization_name: Anthropic
     access: limited
@@ -214,7 +214,7 @@ models:
     tags: [ANTHROPIC_CLAUDE_1_MODEL_TAG, TEXT_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, ABLATION_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
 
   - name: anthropic/claude-2.0
-    display_name: Anthropic Claude 2.0
+    display_name: Claude 2.0
     description: Claude 2.0 is a general purpose large language model developed by Anthropic. It uses a transformer architecture and is trained via unsupervised learning, RLHF, and Constitutional AI (including both a supervised and Reinforcement Learning (RL) phase). ([model card](https://efficient-manatee.files.svdcdn.com/production/images/Model-Card-Claude-2.pdf))
     creator_organization_name: Anthropic
     access: limited
@@ -222,7 +222,7 @@ models:
     tags: [ANTHROPIC_CLAUDE_2_MODEL_TAG, TEXT_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, ABLATION_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
 
   - name: anthropic/claude-2.1
-    display_name: Anthropic Claude 2.1
+    display_name: Claude 2.1
     description: Claude 2.1 is a general purpose large language model developed by Anthropic. It uses a transformer architecture and is trained via unsupervised learning, RLHF, and Constitutional AI (including both a supervised and Reinforcement Learning (RL) phase). ([model card](https://efficient-manatee.files.svdcdn.com/production/images/Model-Card-Claude-2.pdf))
     creator_organization_name: Anthropic
     access: limited
@@ -231,7 +231,7 @@ models:
 
   - name: anthropic/claude-3-haiku-20240307
     display_name: Claude 3 Haiku (20240307)
-    description: Claude 3 is a a family of models that possess vision and multilingual capabilities. They were trained with various methods such as unsupervised learning and Constitutional AI.
+    description: Claude 3 is a a family of models that possess vision and multilingual capabilities. They were trained with various methods such as unsupervised learning and Constitutional AI ([blog](https://www.anthropic.com/news/claude-3-family)).
     creator_organization_name: Anthropic
     access: limited
     release_date: 2024-03-13  # https://www.anthropic.com/news/claude-3-haiku
@@ -239,7 +239,7 @@ models:
 
   - name: anthropic/claude-3-sonnet-20240229
     display_name: Claude 3 Sonnet (20240229)
-    description: Claude 3 is a a family of models that possess vision and multilingual capabilities. They were trained with various methods such as unsupervised learning and Constitutional AI.
+    description: Claude 3 is a a family of models that possess vision and multilingual capabilities. They were trained with various methods such as unsupervised learning and Constitutional AI ([blog](https://www.anthropic.com/news/claude-3-family)).
     creator_organization_name: Anthropic
     access: limited
     release_date: 2024-03-04  # https://www.anthropic.com/news/claude-3-family
@@ -247,9 +247,9 @@ models:
 
   - name: anthropic/claude-3-opus-20240229
     display_name: Claude 3 Opus (20240229)
-    description: Claude 3 is a a family of models that possess vision and multilingual capabilities. They were trained with various methods such as unsupervised learning and Constitutional AI.
-    creator_organization_name: Anthropic
+    description: Claude 3 is a a family of models that possess vision and multilingual capabilities. They were trained with various methods such as unsupervised learning and Constitutional AI ([blog](https://www.anthropic.com/news/claude-3-family)).
     access: limited
+    creator_organization_name: Anthropic
     release_date: 2024-03-04  # https://www.anthropic.com/news/claude-3-family
     tags: [ANTHROPIC_CLAUDE_3_MODEL_TAG, TEXT_MODEL_TAG, VISION_LANGUAGE_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
 
@@ -534,7 +534,7 @@ models:
     access: open
     num_parameters: 132000000000
     release_date: 2024-03-27
-    tags: [TEXT_MODEL_TAG, PARTIAL_FUNCTIONALITY_TEXT_MODEL_TAG]
+    tags: [TEXT_MODEL_TAG, PARTIAL_FUNCTIONALITY_TEXT_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
 
 
   # DeepMind
@@ -559,8 +559,8 @@ models:
 
   # Deepseek
   - name: deepseek-ai/deepseek-llm-67b-chat
-    display_name: DeepSeek Chat (67B)
-    description: DeepSeek Chat is a open-source language model trained on 2 trillion tokens in both English and Chinese, and fine-tuned supervised fine-tuning (SFT) and Direct Preference Optimization (DPO). ([paper](https://arxiv.org/abs/2401.02954))
+    display_name: DeepSeek LLM Chat (67B)
+    description: DeepSeek LLM Chat is a open-source language model trained on 2 trillion tokens in both English and Chinese, and fine-tuned supervised fine-tuning (SFT) and Direct Preference Optimization (DPO). ([paper](https://arxiv.org/abs/2401.02954))
     creator_organization_name: DeepSeek
     access: open
     num_parameters: 67000000000
@@ -670,7 +670,7 @@ models:
     creator_organization_name: Google
     access: limited
     release_date: 2023-12-13
-    tags: [TEXT_MODEL_TAG, GOOGLE_GEMINI_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG]
+    tags: [TEXT_MODEL_TAG, GOOGLE_GEMINI_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
 
   - name: google/gemini-1.0-pro-001
     display_name: Gemini 1.0 Pro
@@ -678,7 +678,7 @@ models:
     creator_organization_name: Google
     access: limited
     release_date: 2023-12-13
-    tags: [TEXT_MODEL_TAG, GOOGLE_GEMINI_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG]
+    tags: [TEXT_MODEL_TAG, GOOGLE_GEMINI_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
 
     # Note: This is aliased to a snapshot of gemini-pro-vision. When possible, please use a versioned snapshot instead.
   - name: google/gemini-pro-vision
@@ -695,15 +695,15 @@ models:
     creator_organization_name: Google
     access: limited
     release_date: 2023-12-13
-    tags: [VISION_LANGUAGE_MODEL_TAG, GOOGLE_GEMINI_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG]
+    tags: [VISION_LANGUAGE_MODEL_TAG, GOOGLE_GEMINI_MODEL_TAG, GOOGLE_GEMINI_PRO_VISION_V1_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG]
 
   - name: google/gemini-1.5-pro-preview-0409
-    display_name: Gemini 1.5 Pro
+    display_name: Gemini 1.5 Pro (0409 preview)
     description: Gemini 1.5 Pro is a multimodal mixture-of-experts model capable of recalling and reasoning over fine-grained information from long contexts. ([paper](https://arxiv.org/abs/2403.05530))
     creator_organization_name: Google
     access: limited
     release_date: 2024-04-10
-    tags: [TEXT_MODEL_TAG, GOOGLE_GEMINI_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG]
+    tags: [TEXT_MODEL_TAG, VISION_LANGUAGE_MODEL_TAG, GOOGLE_GEMINI_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
 
   - name: google/gemma-2b
     display_name: Gemma (2B)
@@ -801,9 +801,18 @@ models:
 
 
   # HuggingFace
+  - name: HuggingFaceM4/idefics2-8b
+    display_name: IDEFICS 2 (8B)
+    description: IDEFICS 2 (8B parameters) is an open multimodal model that accepts arbitrary sequences of image and text inputs and produces text outputs. ([blog](https://huggingface.co/blog/idefics2)).
+    creator_organization_name: HuggingFace
+    access: open
+    num_parameters: 8000000000
+    release_date: 2024-04-15
+    tags: [VISION_LANGUAGE_MODEL_TAG, IDEFICS_MODEL_TAG, FULL_FUNCTIONALITY_VLM_TAG]
+
   - name: HuggingFaceM4/idefics-9b
     display_name: IDEFICS (9B)
-    description: IDEFICS (9B parameters) is an open-source model based on DeepMind's Flamingo. ([blog](https://huggingface.co/blog/idefics))
+    description: IDEFICS (9B parameters) is an open-source model based on DeepMind's Flamingo ([blog](https://huggingface.co/blog/idefics)).
     creator_organization_name: HuggingFace
     access: open
     num_parameters: 9000000000
@@ -811,8 +820,8 @@ models:
     tags: [VISION_LANGUAGE_MODEL_TAG, IDEFICS_MODEL_TAG, FULL_FUNCTIONALITY_VLM_TAG]
 
   - name: HuggingFaceM4/idefics-9b-instruct
-    display_name: IDEFICS instruct (9B)
-    description: IDEFICS instruct (9B parameters) is an open-source model based on DeepMind's Flamingo. ([blog](https://huggingface.co/blog/idefics))
+    display_name: IDEFICS-instruct (9B)
+    description: IDEFICS-instruct (9B parameters) is the instruction-tuned version of IDEFICS 9B ([blog](https://huggingface.co/blog/idefics)).
     creator_organization_name: HuggingFace
     access: open
     num_parameters: 9000000000
@@ -821,7 +830,7 @@ models:
 
   - name: HuggingFaceM4/idefics-80b
     display_name: IDEFICS (80B)
-    description: IDEFICS (80B parameters) is an open-source model based on DeepMind's Flamingo. ([blog](https://huggingface.co/blog/idefics))
+    description: IDEFICS (80B parameters) is an open-source model based on DeepMind's Flamingo ([blog](https://huggingface.co/blog/idefics)).
     creator_organization_name: HuggingFace
     access: open
     num_parameters: 80000000000
@@ -829,8 +838,8 @@ models:
     tags: [VISION_LANGUAGE_MODEL_TAG, IDEFICS_MODEL_TAG, FULL_FUNCTIONALITY_VLM_TAG]
 
   - name: HuggingFaceM4/idefics-80b-instruct
-    display_name: IDEFICS instruct (80B)
-    description: IDEFICS instruct (80B parameters) is an open-source model based on DeepMind's Flamingo. ([blog](https://huggingface.co/blog/idefics))
+    display_name: IDEFICS-instruct (80B)
+    description: IDEFICS-instruct (80B parameters) is the instruction-tuned version of IDEFICS 80B ([blog](https://huggingface.co/blog/idefics)).
     creator_organization_name: HuggingFace
     access: open
     num_parameters: 80000000000
@@ -1210,6 +1219,24 @@ models:
     release_date: 2024-04-18
     tags: [TEXT_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG]
 
+  - name: meta/llama-3-8b-chat
+    display_name: Llama 3 Chat (8B)
+    description: Llama 3 is a family of language models that have been trained on more than 15 trillion tokens, and use Grouped-Query Attention (GQA) for improved inference scalability. It used SFT, rejection sampling, PPO and DPO for post-training.
+    creator_organization_name: Meta
+    access: open
+    num_parameters: 8000000000
+    release_date: 2024-04-18
+    tags: [TEXT_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
+
+  - name: meta/llama-3-70b-chat
+    display_name: Llama 3 Chat (70B)
+    description: Llama 3 is a family of language models that have been trained on more than 15 trillion tokens, and use Grouped-Query Attention (GQA) for improved inference scalability. It used SFT, rejection sampling, PPO and DPO for post-training.
+    creator_organization_name: Meta
+    access: open
+    num_parameters: 70000000000
+    release_date: 2024-04-18
+    tags: [TEXT_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
+
 
   # Microsoft/NVIDIA
   - name: microsoft/TNLGv2_530B
@@ -1247,11 +1274,46 @@ models:
     num_parameters: 13000000000
     release_date: 2023-10-05
     tags: [VISION_LANGUAGE_MODEL_TAG, LLAVA_MODEL_TAG, LIMITED_FUNCTIONALITY_VLM_TAG]
-    
+
+  - name: uw-madison/llava-v1.6-vicuna-7b-hf
+    display_name: LLaVA 1.6 (7B)
+    description: LLaVa is an open-source chatbot trained by fine-tuning LlamA/Vicuna on GPT-generated multimodal instruction-following data. ([paper](https://arxiv.org/abs/2304.08485))
+    creator_organization_name: Microsoft
+    access: open
+    num_parameters: 7000000000
+    release_date: 2024-01-01
+    tags: [VISION_LANGUAGE_MODEL_TAG, LLAVA_MODEL_TAG, LIMITED_FUNCTIONALITY_VLM_TAG]
+
+  - name: uw-madison/llava-v1.6-vicuna-13b-hf
+    display_name: LLaVA 1.6 (13B)
+    description: LLaVa is an open-source chatbot trained by fine-tuning LlamA/Vicuna on GPT-generated multimodal instruction-following data. ([paper](https://arxiv.org/abs/2304.08485))
+    creator_organization_name: Microsoft
+    access: open
+    num_parameters: 13000000000
+    release_date: 2024-01-01
+    tags: [VISION_LANGUAGE_MODEL_TAG, LLAVA_MODEL_TAG, LIMITED_FUNCTIONALITY_VLM_TAG]
+
+  - name: uw-madison/llava-v1.6-mistral-7b-hf
+    display_name: LLaVA 1.6 + Mistral (7B)
+    description: LLaVa is an open-source chatbot trained by fine-tuning LlamA/Vicuna on GPT-generated multimodal instruction-following data. ([paper](https://arxiv.org/abs/2304.08485))
+    creator_organization_name: Microsoft
+    access: open
+    num_parameters: 7000000000
+    release_date: 2024-01-01
+    tags: [ VISION_LANGUAGE_MODEL_TAG, LLAVA_MODEL_TAG, LIMITED_FUNCTIONALITY_VLM_TAG ]
+
+  - name: uw-madison/llava-v1.6-34b-hf
+    display_name: LLaVA + Nous-Hermes-2-Yi-34B (34B)
+    description: LLaVa is an open-source chatbot trained by fine-tuning LlamA/Vicuna on GPT-generated multimodal instruction-following data. ([paper](https://arxiv.org/abs/2304.08485))
+    creator_organization_name: Microsoft
+    access: open
+    num_parameters: 34000000000
+    release_date: 2024-01-01
+    tags: [VISION_LANGUAGE_MODEL_TAG, LLAVA_MODEL_TAG, LIMITED_FUNCTIONALITY_VLM_TAG]
   
   - name: openflamingo/OpenFlamingo-9B-vitl-mpt7b
     display_name: OpenFlamingo (9B)
-    description: OpenFlamingo is an open source implementation of DeepMind's Flamingo models. This 9B-parameter model uses a CLIP ViT-L/14 vision encoder and MPT-7B language model. ([paper](https://arxiv.org/abs/2308.01390))
+    description: OpenFlamingo is an open source implementation of DeepMind's Flamingo models. This 9B-parameter model uses a CLIP ViT-L/14 vision encoder and MPT-7B language model ([paper](https://arxiv.org/abs/2308.01390)).
     creator_organization_name: OpenFlamingo
     access: open
     num_parameters: 9000000000
@@ -1286,7 +1348,22 @@ models:
     num_parameters: 34000000000
     release_date: 2023-11-02
     tags: [TEXT_MODEL_TAG, FULL_FUNCTIONALITY_TEXT_MODEL_TAG]
-
+  - name: 01-ai/yi-6b-chat
+    display_name: Yi Chat (6B)
+    description: The Yi models are large language models trained from scratch by developers at 01.AI.
+    creator_organization_name: 01.AI
+    access: open
+    num_parameters: 6000000000
+    release_date: 2023-11-23
+    tags: [TEXT_MODEL_TAG, FULL_FUNCTIONALITY_TEXT_MODEL_TAG]
+  - name: 01-ai/yi-34b-chat
+    display_name: Yi Chat (34B)
+    description: The Yi models are large language models trained from scratch by developers at 01.AI.
+    creator_organization_name: 01.AI
+    access: open
+    num_parameters: 34000000000
+    release_date: 2023-11-23
+    tags: [TEXT_MODEL_TAG, FULL_FUNCTIONALITY_TEXT_MODEL_TAG]
 
   # Allen Institute for AI
   # OLMo Blog: https://blog.allenai.org/olmo-open-language-model-87ccfc95f580
@@ -1350,7 +1427,16 @@ models:
 
   - name: mistralai/mixtral-8x22b
     display_name: Mixtral (8x22B)
-    description: Mistral AI's mixture-of-experts model ([tweet](https://twitter.com/MistralAI/status/1777869263778291896)).
+    description: Mistral AI's mixture-of-experts model that uses 39B active parameters out of 141B ([blog post](https://mistral.ai/news/mixtral-8x22b/)).
+    creator_organization_name: Mistral AI
+    access: open
+    num_parameters: 176000000000
+    release_date: 2024-04-10
+    tags: [TEXT_MODEL_TAG, PARTIAL_FUNCTIONALITY_TEXT_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
+
+  - name: mistralai/mixtral-8x22b-instruct-v0.1
+    display_name: Mixtral Instruct (8x22B)
+    description: Mistral AI's mixture-of-experts model that uses 39B active parameters out of 141B ([blog post](https://mistral.ai/news/mixtral-8x22b/)).
     creator_organization_name: Mistral AI
     access: open
     num_parameters: 176000000000
@@ -1721,8 +1807,17 @@ models:
     tags: [TEXT_MODEL_TAG, OPENAI_CHATGPT_MODEL_TAG, LIMITED_FUNCTIONALITY_TEXT_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
 
   - name: openai/gpt-4-vision-preview
-    display_name: GPT-4V (preview)
-    description: GPT-4V is a large multimodal model that accepts both text and images and is optimized for chat but works well for traditional completions tasks.
+    # According to https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4, this model has pointed gpt-4-1106-vision-preview.
+    display_name: GPT-4V (1106 preview)
+    description: GPT-4V is a large multimodal model that accepts both text and images and is optimized for chat ([model card](https://openai.com/research/gpt-4v-system-card)).
+    creator_organization_name: OpenAI
+    access: limited
+    release_date: 2023-11-06
+    tags: [VISION_LANGUAGE_MODEL_TAG, OPENAI_CHATGPT_MODEL_TAG, FULL_FUNCTIONALITY_VLM_TAG]
+
+  - name: openai/gpt-4-1106-vision-preview
+    display_name: GPT-4V (1106 preview)
+    description: GPT-4V is a large multimodal model that accepts both text and images and is optimized for chat ([model card](https://openai.com/research/gpt-4v-system-card)).
     creator_organization_name: OpenAI
     access: limited
     release_date: 2023-11-06
@@ -1898,7 +1993,7 @@ models:
 
   - name: qwen/qwen-vl
     display_name: Qwen-VL
-    description: Visual multimodal version of the large model series ([paper](https://arxiv.org/abs/2308.12966)).
+    description: Visual multimodal version of the Qwen large language model series ([paper](https://arxiv.org/abs/2308.12966)).
     creator_organization_name: Alibaba Cloud
     access: open
     release_date: 2023-08-24
@@ -1906,7 +2001,7 @@ models:
 
   - name: qwen/qwen-vl-chat
     display_name: Qwen-VL Chat
-    description: Chat version of the visual multimodal model Qwen ([paper](https://arxiv.org/abs/2308.12966)).
+    description: Chat version of Qwen-VL ([paper](https://arxiv.org/abs/2308.12966)).
     creator_organization_name: Alibaba Cloud
     access: open
     release_date: 2023-08-24
@@ -1923,6 +2018,16 @@ models:
     tags: [] # TODO: add tags
 
 
+  # Snowflake
+  - name: snowflake/snowflake-arctic-instruct
+    display_name: Arctic Instruct
+    description: Arctic combines a 10B dense transformer model with a residual 128x3.66B MoE MLP resulting in 480B total and 17B active parameters chosen using a top-2 gating.
+    creator_organization_name: Snowflake
+    access: open
+    num_parameters: 482000000000
+    release_date: 2024-04-24
+    tags: [TEXT_MODEL_TAG, PARTIAL_FUNCTIONALITY_TEXT_MODEL_TAG, INSTRUCTION_FOLLOWING_MODEL_TAG]
+
 
   # Stability AI
   - name: stabilityai/stablelm-base-alpha-3b

helm/config/tokenizer_configs.yaml

@@ -331,6 +331,13 @@ tokenizer_configs:
     end_of_text_token: "<|endoftext|>"
     prefix_token: ""
 
+  # Snowflake
+  - name: snowflake/snowflake-arctic-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|im_end|>"
+    prefix_token: "<|im_start|>"
+
   # Tiiuae
   - name: tiiuae/falcon-7b
     tokenizer_spec: