palimpzest 0.7.21__py3-none-any.whl → 0.8.1__py3-none-any.whl

palimpzest/constants.py

@@ -10,77 +10,125 @@ class Model(str, Enum):
     which requires invoking an LLM. It does NOT specify whether the model need be executed
     remotely or locally (if applicable).
     """
-    LLAMA3_2_3B = "meta-llama/Llama-3.2-3B-Instruct-Turbo"
-    LLAMA3_1_8B = "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"
-    LLAMA3_3_70B = "meta-llama/Llama-3.3-70B-Instruct-Turbo"
-    LLAMA3_2_90B_V = "meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo"
-    MIXTRAL = "mistralai/Mixtral-8x7B-Instruct-v0.1"
-    DEEPSEEK_V3 = "deepseek-ai/DeepSeek-V3"
-    DEEPSEEK_R1_DISTILL_QWEN_1_5B = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
-    GPT_4o = "gpt-4o-2024-08-06"
-    GPT_4o_MINI = "gpt-4o-mini-2024-07-18"
+    LLAMA3_2_3B = "together_ai/meta-llama/Llama-3.2-3B-Instruct-Turbo"
+    LLAMA3_1_8B = "together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"
+    LLAMA3_3_70B = "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"
+    LLAMA3_2_90B_V = "together_ai/meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo"
+    DEEPSEEK_V3 = "together_ai/deepseek-ai/DeepSeek-V3"
+    DEEPSEEK_R1_DISTILL_QWEN_1_5B = "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
+    GPT_4o = "openai/gpt-4o-2024-08-06"
+    GPT_4o_MINI = "openai/gpt-4o-mini-2024-07-18"
+    GPT_4_1 = "openai/gpt-4.1-2025-04-14"
+    GPT_4_1_MINI = "openai/gpt-4.1-mini-2025-04-14"
+    GPT_4_1_NANO = "openai/gpt-4.1-nano-2025-04-14"
+    GPT_5 = "openai/gpt-5-2025-08-07"
+    GPT_5_MINI = "openai/gpt-5-mini-2025-08-07"
+    GPT_5_NANO = "openai/gpt-5-nano-2025-08-07"
+    o4_MINI = "openai/o4-mini-2025-04-16"  # noqa: N815
     TEXT_EMBEDDING_3_SMALL = "text-embedding-3-small"
     CLIP_VIT_B_32 = "clip-ViT-B-32"
+    CLAUDE_3_5_SONNET = "anthropic/claude-3-5-sonnet-20241022"
+    CLAUDE_3_7_SONNET = "anthropic/claude-3-7-sonnet-20250219"
+    CLAUDE_3_5_HAIKU = "anthropic/claude-3-5-haiku-20241022"
+    GEMINI_2_0_FLASH = "vertex_ai/gemini-2.0-flash"
+    GEMINI_2_5_FLASH = "vertex_ai/gemini-2.5-flash"
+    GEMINI_2_5_PRO = "vertex_ai/gemini-2.5-pro"
+    GOOGLE_GEMINI_2_5_FLASH = "google/gemini-2.5-flash"
+    GOOGLE_GEMINI_2_5_FLASH_LITE = "google/gemini-2.5-flash-lite"
+    GOOGLE_GEMINI_2_5_PRO = "google/gemini-2.5-pro"
+    LLAMA_4_MAVERICK = "vertex_ai/meta/llama-4-maverick-17b-128e-instruct-maas"
+    GPT_4o_AUDIO_PREVIEW = "openai/gpt-4o-audio-preview"
+    GPT_4o_MINI_AUDIO_PREVIEW = "openai/gpt-4o-mini-audio-preview"
+    VLLM_QWEN_1_5_0_5B_CHAT = "hosted_vllm/qwen/Qwen1.5-0.5B-Chat"
     # o1 = "o1-2024-12-17"
 
     def __repr__(self):
         return f"{self.name}"
 
-    def is_deepseek_model(self):
-        return "deepseek" in self.value.lower()
-
     def is_llama_model(self):
         return "llama" in self.value.lower()
 
-    def is_mixtral_model(self):
-        return "mixtral" in self.value.lower()
-
     def is_clip_model(self):
         return "clip" in self.value.lower()
 
     def is_together_model(self):
-        is_llama_model = self.is_llama_model()
-        is_mixtral_model = self.is_mixtral_model()
-        is_deepseek_model = self.is_deepseek_model()
-        is_clip_model = self.is_clip_model()
-        return is_llama_model or is_mixtral_model or is_deepseek_model or is_clip_model
-
-    def is_gpt_4o_model(self):
-        return "gpt-4o" in self.value.lower()
-
-    def is_o1_model(self):
-        return "o1" in self.value.lower()
+        return "together_ai" in self.value.lower() or self.is_clip_model()
 
     def is_text_embedding_model(self):
         return "text-embedding" in self.value.lower()
 
+    def is_o_model(self):
+        return self in [Model.o4_MINI]
+
+    def is_gpt_5_model(self):
+        return self in [Model.GPT_5, Model.GPT_5_MINI, Model.GPT_5_NANO]
+
     def is_openai_model(self):
-        is_gpt4_model = self.is_gpt_4o_model()
-        is_o1_model = self.is_o1_model()
-        is_text_embedding_model = self.is_text_embedding_model()
-        return is_gpt4_model or is_o1_model or is_text_embedding_model
+        return "openai" in self.value.lower() or self.is_text_embedding_model()
+
+    def is_anthropic_model(self):
+        return "anthropic" in self.value.lower()
+
+    def is_vertex_model(self):
+        return "vertex_ai" in self.value.lower()
+
+    def is_google_model(self):
+        return "google" in self.value.lower()
+
+    def is_vllm_model(self):
+        return "hosted_vllm" in self.value.lower()
 
+    def is_reasoning_model(self):
+        reasoning_models = [
+            Model.GPT_5, Model.GPT_5_MINI, Model.GPT_5_NANO, Model.o4_MINI,
+            Model.GEMINI_2_5_PRO, Model.GEMINI_2_5_FLASH,
+            Model.GOOGLE_GEMINI_2_5_PRO, Model.GOOGLE_GEMINI_2_5_FLASH, Model.GOOGLE_GEMINI_2_5_FLASH_LITE,
+            Model.CLAUDE_3_7_SONNET,
+        ]
+        return self in reasoning_models
+
+    def is_text_model(self):
+        non_text_models = [
+            Model.LLAMA3_2_90B_V,
+            Model.CLIP_VIT_B_32, Model.TEXT_EMBEDDING_3_SMALL,
+            Model.GPT_4o_AUDIO_PREVIEW, Model.GPT_4o_MINI_AUDIO_PREVIEW,
+        ]
+        return self not in non_text_models
+
+    # TODO: I think SONNET and HAIKU are vision-capable too
     def is_vision_model(self):
-        vision_models = [
-            "meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo",
-            "gpt-4o-2024-08-06",
-            "gpt-4o-mini-2024-07-18",
-            "o1-2024-12-17",
+        return self in [
+            Model.LLAMA3_2_90B_V, Model.LLAMA_4_MAVERICK,
+            Model.GPT_4o, Model.GPT_4o_MINI, Model.GPT_4_1, Model.GPT_4_1_MINI, Model.GPT_4_1_NANO, Model.o4_MINI, Model.GPT_5, Model.GPT_5_MINI, Model.GPT_5_NANO,
+            Model.GEMINI_2_0_FLASH, Model.GEMINI_2_5_FLASH, Model.GEMINI_2_5_PRO,
+            Model.GOOGLE_GEMINI_2_5_PRO, Model.GOOGLE_GEMINI_2_5_FLASH, Model.GOOGLE_GEMINI_2_5_FLASH_LITE,
         ]
-        return self.value in vision_models
 
-    def is_embedding_model(self):
-        is_clip_model = self.is_clip_model()
-        is_text_embedding_model = self.is_text_embedding_model()
-        return is_clip_model or is_text_embedding_model
+    def is_audio_model(self):
+        return self in [
+            Model.GPT_4o_AUDIO_PREVIEW, Model.GPT_4o_MINI_AUDIO_PREVIEW,
+            Model.GEMINI_2_0_FLASH, Model.GEMINI_2_5_FLASH, Model.GEMINI_2_5_PRO,
+            Model.GOOGLE_GEMINI_2_5_PRO, Model.GOOGLE_GEMINI_2_5_FLASH, Model.GOOGLE_GEMINI_2_5_FLASH_LITE,
+        ]
 
-class APIClient(str, Enum):
-    """
-    APIClient describes the API client to be used when invoking an LLM.
-    """
+    def is_text_image_multimodal_model(self):
+        return self in [
+            Model.LLAMA_4_MAVERICK,
+            Model.GPT_4o, Model.GPT_4o_MINI, Model.GPT_4_1, Model.GPT_4_1_MINI, Model.GPT_4_1_NANO, Model.o4_MINI, Model.GPT_5, Model.GPT_5_MINI, Model.GPT_5_NANO,
+            Model.GEMINI_2_0_FLASH, Model.GEMINI_2_5_FLASH, Model.GEMINI_2_5_PRO,
+            Model.GOOGLE_GEMINI_2_5_PRO, Model.GOOGLE_GEMINI_2_5_FLASH, Model.GOOGLE_GEMINI_2_5_FLASH_LITE,
+        ]
+
+    def is_text_audio_multimodal_model(self):
+        return self in [
+            Model.GPT_4o_AUDIO_PREVIEW, Model.GPT_4o_MINI_AUDIO_PREVIEW,
+            Model.GEMINI_2_0_FLASH, Model.GEMINI_2_5_FLASH, Model.GEMINI_2_5_PRO,
+            Model.GOOGLE_GEMINI_2_5_PRO, Model.GOOGLE_GEMINI_2_5_FLASH, Model.GOOGLE_GEMINI_2_5_FLASH_LITE,
+        ]
+
+    def is_embedding_model(self):
+        return self in [Model.CLIP_VIT_B_32, Model.TEXT_EMBEDDING_3_SMALL]
 
-    OPENAI = "openai"
-    TOGETHER = "together"
 
 class PromptStrategy(str, Enum):
     """
@@ -90,28 +138,48 @@ class PromptStrategy(str, Enum):
 
     # Chain-of-Thought Boolean Prompt Strategies
     COT_BOOL = "chain-of-thought-bool"
+    COT_BOOL_NO_REASONING = "chain-of-thought-bool-no-reasoning"
     # COT_BOOL_CRITIC = "chain-of-thought-bool-critic"
     # COT_BOOL_REFINE = "chain-of-thought-bool-refine"
 
     # Chain-of-Thought Boolean with Image Prompt Strategies
     COT_BOOL_IMAGE = "chain-of-thought-bool-image"
+    COT_BOOL_IMAGE_NO_REASONING = "chain-of-thought-bool-image"
+    COT_BOOL_AUDIO = "chain-of-thought-bool-audio"
+    COT_BOOL_AUDIO_NO_REASONING = "chain-of-thought-bool-audio"
     # COT_BOOL_IMAGE_CRITIC = "chain-of-thought-bool-image-critic"
     # COT_BOOL_IMAGE_REFINE = "chain-of-thought-bool-image-refine"
 
+    # Chain-of-Thought Join Prompt Strategies
+    COT_JOIN = "chain-of-thought-join"
+    COT_JOIN_NO_REASONING = "chain-of-thought-join-no-reasoning"
+    COT_JOIN_IMAGE = "chain-of-thought-join-image"
+    COT_JOIN_IMAGE_NO_REASONING = "chain-of-thought-join-image-no-reasoning"
+    COT_JOIN_AUDIO = "chain-of-thought-join-audio"
+    COT_JOIN_AUDIO_NO_REASONING = "chain-of-thought-join-audio-no-reasoning"
+
     # Chain-of-Thought Question Answering Prompt Strategies
     COT_QA = "chain-of-thought-question"
+    COT_QA_NO_REASONING = "chain-of-thought-question-no-reasoning"
     COT_QA_CRITIC = "chain-of-thought-question-critic"
     COT_QA_REFINE = "chain-of-thought-question-refine"
 
     # Chain-of-Thought Question with Image Prompt Strategies
     COT_QA_IMAGE = "chain-of-thought-question-image"
+    COT_QA_IMAGE_NO_REASONING = "chain-of-thought-question-image-no-reasoning"
     COT_QA_IMAGE_CRITIC = "chain-of-thought-question-critic-image"
     COT_QA_IMAGE_REFINE = "chain-of-thought-question-refine-image"
 
+    # Chain-of-Thought Queestion with Audio Prompt Strategies
+    COT_QA_AUDIO = "chain-of-thought-question-audio"
+    COT_QA_AUDIO_NO_REASONING = "chain-of-thought-question-audio-no-reasoning"
+    # TODO: COT_QA_AUDIO_CRITIC/REFINE
+
     # Mixture-of-Agents Prompt Strategies
     COT_MOA_PROPOSER = "chain-of-thought-mixture-of-agents-proposer"
     COT_MOA_PROPOSER_IMAGE = "chain-of-thought-mixture-of-agents-proposer-image"
     COT_MOA_AGG = "chain-of-thought-mixture-of-agents-aggregation"
+    # TODO: COT_MOA_PROPOSER_AUDIO 
 
     # Split Convert Prompt Strategies
     SPLIT_PROPOSER = "split-proposer"
@@ -120,11 +188,17 @@ class PromptStrategy(str, Enum):
     def is_image_prompt(self):
         return "image" in self.value
 
+    def is_audio_prompt(self):
+        return "audio" in self.value
+
     def is_bool_prompt(self):
         return "bool" in self.value
 
+    def is_join_prompt(self):
+        return "join" in self.value
+
     def is_convert_prompt(self):
-        return "bool" not in self.value
+        return "bool" not in self.value and "join" not in self.value
 
     def is_critic_prompt(self):
         return "critic" in self.value
@@ -144,6 +218,9 @@ class PromptStrategy(str, Enum):
     def is_split_merger_prompt(self):
         return "split-merger" in self.value
 
+    def is_no_reasoning_prompt(self):
+        return "no-reasoning" in self.value
+
 class AggFunc(str, Enum):
     COUNT = "count"
     AVERAGE = "average"
@@ -169,6 +246,7 @@ class PickOutputStrategy(str, Enum):
     ENSEMBLE = "ensemble"
 
 
+AUDIO_EXTENSIONS = [".wav"]
 IMAGE_EXTENSIONS = [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff"]
 PDF_EXTENSIONS = [".pdf"]
 XLS_EXTENSIONS = [".xls", ".xlsx"]
@@ -210,8 +288,8 @@ NAIVE_BYTES_PER_RECORD = 1024
 # Rough conversion from # of characters --> # of tokens; assumes 1 token ~= 4 chars
 TOKENS_PER_CHARACTER = 0.25
 
-# Rough estimate of the number of tokens the context is allowed to take up for MIXTRAL and LLAMA3 models
-MIXTRAL_LLAMA_CONTEXT_TOKENS_LIMIT = 6000
+# Rough estimate of the number of tokens the context is allowed to take up for LLAMA3 models
+LLAMA_CONTEXT_TOKENS_LIMIT = 6000
 
 # a naive estimate for the input record size
 NAIVE_EST_SOURCE_RECORD_SIZE_IN_BYTES = 1_000_000
@@ -219,6 +297,9 @@ NAIVE_EST_SOURCE_RECORD_SIZE_IN_BYTES = 1_000_000
 # a naive estimate for filter selectivity
 NAIVE_EST_FILTER_SELECTIVITY = 0.5
 
+# a naive estimate for join selectivity
+NAIVE_EST_JOIN_SELECTIVITY = 0.5
+
 # a naive estimate for the number of input tokens processed per record
 NAIVE_EST_NUM_INPUT_TOKENS = 1000
 
@@ -246,6 +327,7 @@ LOG_LLM_OUTPUT = False
 # - in the future we should split quality for vision vs. multi-modal vs. text
 # - code quality was computed using HumanEval, but that benchmark is too easy and should be replaced.
 # - https://huggingface.co/spaces/TIGER-Lab/MMLU-Pro
+# - https://www.vals.ai/benchmarks/mmlu_pro-08-12-2025
 #
 # Cost is presented in terms of USD / token for input tokens and USD / token for
 # generated tokens.
@@ -260,75 +342,74 @@ LLAMA3_2_3B_INSTRUCT_MODEL_CARD = {
     "usd_per_input_token": 0.06 / 1e6,
     "usd_per_output_token": 0.06 / 1e6,
     ##### Time #####
-    "seconds_per_output_token": 0.0064,
+    "seconds_per_output_token": 0.0079,
     ##### Agg. Benchmark #####
     "overall": 36.50, # https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct/discussions/13
-    ##### Code #####
-    "code": 0.0,
 }
 LLAMA3_1_8B_INSTRUCT_MODEL_CARD = {
     ##### Cost in USD #####
     "usd_per_input_token": 0.18 / 1e6,
     "usd_per_output_token": 0.18 / 1e6,
     ##### Time #####
-    "seconds_per_output_token": 0.0059,
+    "seconds_per_output_token": 0.0050,
     ##### Agg. Benchmark #####
     "overall": 44.25,
-    ##### Code #####
-    "code": 72.6,
 }
 LLAMA3_3_70B_INSTRUCT_MODEL_CARD = {
     ##### Cost in USD #####
     "usd_per_input_token": 0.88 / 1e6,
     "usd_per_output_token": 0.88 / 1e6,
     ##### Time #####
-    "seconds_per_output_token": 0.0139,
+    "seconds_per_output_token": 0.0122,
     ##### Agg. Benchmark #####
-    "overall": 65.92,
-    ##### Code #####
-    "code": 88.4,
+    "overall": 69.9,
 }
 LLAMA3_2_90B_V_MODEL_CARD = {
     ##### Cost in USD #####
     "usd_per_input_token": 1.2 / 1e6,
     "usd_per_output_token": 1.2 / 1e6,
     ##### Time #####
-    "seconds_per_output_token": 0.0222,
+    "seconds_per_output_token": 0.0303,
     ##### Agg. Benchmark #####
     "overall": 65.00, # set to be slightly higher than gpt-4o-mini
 }
-MIXTRAL_8X_7B_MODEL_CARD = {
-    ##### Cost in USD #####
-    "usd_per_input_token": 0.6 / 1e6,
-    "usd_per_output_token": 0.6 / 1e6,
-    ##### Time #####
-    "seconds_per_output_token": 0.0112,
-    ##### Agg. Benchmark #####
-    "overall": 43.27,
-    ##### Code #####
-    "code": 40.0,
-}
 DEEPSEEK_V3_MODEL_CARD = {
     ##### Cost in USD #####
     "usd_per_input_token": 1.25 / 1E6,
     "usd_per_output_token": 1.25 / 1E6,
     ##### Time #####
-    "seconds_per_output_token": 0.0769,
+    "seconds_per_output_token": 0.0114,
     ##### Agg. Benchmark #####
-    "overall": 75.87,
-    ##### Code #####
-    "code": 92.0,
+    "overall": 73.8,
 }
 DEEPSEEK_R1_DISTILL_QWEN_1_5B_MODEL_CARD = {
     ##### Cost in USD #####
     "usd_per_input_token": 0.18 / 1E6,
     "usd_per_output_token": 0.18 / 1E6,
     ##### Time #####
-    "seconds_per_output_token": 0.0026,
+    "seconds_per_output_token": 0.0050, # NOTE: copied to be same as LLAMA3_1_8B_INSTRUCT_MODEL_CARD; need to update when we have data
     ##### Agg. Benchmark #####
     "overall": 39.90, # https://www.reddit.com/r/LocalLLaMA/comments/1iserf9/deepseek_r1_distilled_models_mmlu_pro_benchmarks/
-    ##### Code #####
-    "code": 0.0,
+}
+GPT_4o_AUDIO_PREVIEW_MODEL_CARD = {
+    # NOTE: COPYING OVERALL AND SECONDS_PER_OUTPUT_TOKEN FROM GPT_4o; need to update when we have audio-specific benchmarks
+    ##### Cost in USD #####
+    "usd_per_audio_input_token": 2.5 / 1e6,
+    "usd_per_output_token": 10.0 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0080,
+    ##### Agg. Benchmark #####
+    "overall": 74.1,
+}
+GPT_4o_MINI_AUDIO_PREVIEW_MODEL_CARD = {
+    # NOTE: COPYING OVERALL AND SECONDS_PER_OUTPUT_TOKEN FROM GPT_4o; need to update when we have audio-specific benchmarks
+    ##### Cost in USD #####
+    "usd_per_audio_input_token": 0.15 / 1e6,
+    "usd_per_output_token": 0.6 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0159,
+    ##### Agg. Benchmark #####
+    "overall": 62.7,
 }
 GPT_4o_MODEL_CARD = {
     # NOTE: it is unclear if the same ($ / token) costs can be applied for vision, or if we have to calculate this ourselves
@@ -336,11 +417,9 @@ GPT_4o_MODEL_CARD = {
     "usd_per_input_token": 2.5 / 1e6,
     "usd_per_output_token": 10.0 / 1e6,
     ##### Time #####
-    "seconds_per_output_token": 0.0079,
+    "seconds_per_output_token": 0.0080,
     ##### Agg. Benchmark #####
-    "overall": 74.68,
-    ##### Code #####
-    "code": 90.0,
+    "overall": 74.1,
 }
 GPT_4o_MINI_MODEL_CARD = {
     # NOTE: it is unclear if the same ($ / token) costs can be applied for vision, or if we have to calculate this ourselves
@@ -348,24 +427,90 @@ GPT_4o_MINI_MODEL_CARD = {
     "usd_per_input_token": 0.15 / 1e6,
     "usd_per_output_token": 0.6 / 1e6,
     ##### Time #####
-    "seconds_per_output_token": 0.0098,
+    "seconds_per_output_token": 0.0159,
+    ##### Agg. Benchmark #####
+    "overall": 62.7,
+}
+GPT_4_1_MODEL_CARD = {
+    # NOTE: it is unclear if the same ($ / token) costs can be applied for vision, or if we have to calculate this ourselves
+    ##### Cost in USD #####
+    "usd_per_input_token": 2.0 / 1e6,
+    "usd_per_output_token": 8.0 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0076,
+    ##### Agg. Benchmark #####
+    "overall": 80.5,
+}
+GPT_4_1_MINI_MODEL_CARD = {
+    # NOTE: it is unclear if the same ($ / token) costs can be applied for vision, or if we have to calculate this ourselves
+    ##### Cost in USD #####
+    "usd_per_input_token": 0.4 / 1e6,
+    "usd_per_output_token": 1.6 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0161,
     ##### Agg. Benchmark #####
-    "overall": 63.09,
-    ##### Code #####
-    "code": 86.0,
+    "overall": 77.2,
 }
-o1_MODEL_CARD = {  # noqa: N816
+GPT_4_1_NANO_MODEL_CARD = {
     # NOTE: it is unclear if the same ($ / token) costs can be applied for vision, or if we have to calculate this ourselves
     ##### Cost in USD #####
-    "usd_per_input_token": 15 / 1e6,
-    "usd_per_output_token": 60 / 1e6,
+    "usd_per_input_token": 0.1 / 1e6,
+    "usd_per_output_token": 0.4 / 1e6,
     ##### Time #####
-    "seconds_per_output_token": 0.0110,
+    "seconds_per_output_token": 0.0060,
     ##### Agg. Benchmark #####
-    "overall": 89.30,
-    ##### Code #####
-    "code": 92.3, # NOTE: just copying MMLU score for now
+    "overall": 62.3,
 }
+GPT_5_MODEL_CARD = {
+    # NOTE: it is unclear if the same ($ / token) costs can be applied for vision, or if we have to calculate this ourselves
+    ##### Cost in USD #####
+    "usd_per_input_token": 1.25 / 1e6,
+    "usd_per_output_token": 10.0 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0060,
+    ##### Agg. Benchmark #####
+    "overall": 87.00,
+}
+GPT_5_MINI_MODEL_CARD = {
+    # NOTE: it is unclear if the same ($ / token) costs can be applied for vision, or if we have to calculate this ourselves
+    ##### Cost in USD #####
+    "usd_per_input_token": 0.25 / 1e6,
+    "usd_per_output_token": 2.0 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0135,
+    ##### Agg. Benchmark #####
+    "overall": 82.50,
+}
+GPT_5_NANO_MODEL_CARD = {
+    # NOTE: it is unclear if the same ($ / token) costs can be applied for vision, or if we have to calculate this ourselves
+    ##### Cost in USD #####
+    "usd_per_input_token": 0.05 / 1e6,
+    "usd_per_output_token": 0.4 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0055,
+    ##### Agg. Benchmark #####
+    "overall": 77.9,
+}
+o4_MINI_MODEL_CARD = {  # noqa: N816
+    # NOTE: it is unclear if the same ($ / token) costs can be applied for vision, or if we have to calculate this ourselves
+    ##### Cost in USD #####
+    "usd_per_input_token": 1.1 / 1e6,
+    "usd_per_output_token": 4.4 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0092,
+    ##### Agg. Benchmark #####
+    "overall": 80.6,  # using number reported for o3-mini; true number is likely higher
+}
+# o1_MODEL_CARD = {  # noqa: N816
+#     # NOTE: it is unclear if the same ($ / token) costs can be applied for vision, or if we have to calculate this ourselves
+#     ##### Cost in USD #####
+#     "usd_per_input_token": 15 / 1e6,
+#     "usd_per_output_token": 60 / 1e6,
+#     ##### Time #####
+#     "seconds_per_output_token": 0.0110,
+#     ##### Agg. Benchmark #####
+#     "overall": 83.50,
+# }
 TEXT_EMBEDDING_3_SMALL_MODEL_CARD = {
     ##### Cost in USD #####
     "usd_per_input_token": 0.02 / 1e6,
@@ -384,7 +529,91 @@ CLIP_VIT_B_32_MODEL_CARD = {
     ##### Agg. Benchmark #####
     "overall": 63.3,  # NOTE: ImageNet top-1 accuracy
 }
-
+CLAUDE_3_5_SONNET_MODEL_CARD = {
+    ##### Cost in USD #####
+    "usd_per_input_token": 3.0 / 1e6,
+    "usd_per_output_token": 15.0 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0154,
+    ##### Agg. Benchmark #####
+    "overall": 78.4,
+}
+CLAUDE_3_7_SONNET_MODEL_CARD = {
+    ##### Cost in USD #####
+    "usd_per_input_token": 3.0 / 1e6,
+    "usd_per_output_token": 15.0 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0156,
+    ##### Agg. Benchmark #####
+    "overall": 80.7,
+}
+CLAUDE_3_5_HAIKU_MODEL_CARD = {
+    ##### Cost in USD #####
+    "usd_per_input_token": 0.8 / 1e6,
+    "usd_per_output_token": 4.0 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0189,
+    ##### Agg. Benchmark #####
+    "overall": 64.1,
+}
+GEMINI_2_0_FLASH_MODEL_CARD = {
+    ##### Cost in USD #####
+    "usd_per_input_token": 0.15 / 1e6,
+    "usd_per_output_token": 0.6 / 1e6,
+    "usd_per_audio_input_token": 1.0 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0054,
+    ##### Agg. Benchmark #####
+    "overall": 77.40,
+}
+GEMINI_2_5_FLASH_LITE_MODEL_CARD = {
+    ##### Cost in USD #####
+    "usd_per_input_token": 0.1 / 1e6,
+    "usd_per_output_token": 0.4 / 1e6,
+    "usd_per_audio_input_token": 0.3 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0034,
+    ##### Agg. Benchmark #####
+    "overall": 79.1, # NOTE: interpolated between gemini 2.5 flash and gemini 2.0 flash
+}
+GEMINI_2_5_FLASH_MODEL_CARD = {
+    ##### Cost in USD #####
+    "usd_per_input_token": 0.30 / 1e6,
+    "usd_per_output_token": 2.5 / 1e6,
+    "usd_per_audio_input_token": 1.0 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0044,
+    ##### Agg. Benchmark #####
+    "overall": 80.75, # NOTE: interpolated between gemini 2.0 flash and gemini 2.5 pro
+}
+GEMINI_2_5_PRO_MODEL_CARD = {
+    ##### Cost in USD #####
+    "usd_per_input_token": 1.25 / 1e6,
+    "usd_per_output_token": 10.0 / 1e6,
+    "usd_per_audio_input_token": 1.25 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0072,
+    ##### Agg. Benchmark #####
+    "overall": 84.10,
+}
+LLAMA_4_MAVERICK_MODEL_CARD = {
+    ##### Cost in USD #####
+    "usd_per_input_token": 0.35 / 1e6,
+    "usd_per_output_token": 1.15 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.0122,
+    ##### Agg. Benchmark #####
+    "overall": 79.4,
+}
+VLLM_QWEN_1_5_0_5B_CHAT_MODEL_CARD = {
+    ##### Cost in USD #####
+    "usd_per_input_token": 0.0 / 1e6,
+    "usd_per_output_token": 0.0 / 1e6,
+    ##### Time #####
+    "seconds_per_output_token": 0.1000, # TODO: fill-in with a better estimate
+    ##### Agg. Benchmark #####
+    "overall": 30.0, # TODO: fill-in with a better estimate
+}
 
 MODEL_CARDS = {
     Model.LLAMA3_2_3B.value: LLAMA3_2_3B_INSTRUCT_MODEL_CARD,
@@ -393,124 +622,29 @@ MODEL_CARDS = {
     Model.LLAMA3_2_90B_V.value: LLAMA3_2_90B_V_MODEL_CARD,
     Model.DEEPSEEK_V3.value: DEEPSEEK_V3_MODEL_CARD,
     Model.DEEPSEEK_R1_DISTILL_QWEN_1_5B.value: DEEPSEEK_R1_DISTILL_QWEN_1_5B_MODEL_CARD,
-    Model.MIXTRAL.value: MIXTRAL_8X_7B_MODEL_CARD,
     Model.GPT_4o.value: GPT_4o_MODEL_CARD,
     Model.GPT_4o_MINI.value: GPT_4o_MINI_MODEL_CARD,
+    Model.GPT_4o_AUDIO_PREVIEW.value: GPT_4o_AUDIO_PREVIEW_MODEL_CARD,
+    Model.GPT_4o_MINI_AUDIO_PREVIEW.value: GPT_4o_MINI_AUDIO_PREVIEW_MODEL_CARD,
+    Model.GPT_4_1.value: GPT_4_1_MODEL_CARD,
+    Model.GPT_4_1_MINI.value: GPT_4_1_MINI_MODEL_CARD,
+    Model.GPT_4_1_NANO.value: GPT_4_1_NANO_MODEL_CARD,
+    Model.GPT_5.value: GPT_5_MODEL_CARD,
+    Model.GPT_5_MINI.value: GPT_5_MINI_MODEL_CARD,
+    Model.GPT_5_NANO.value: GPT_5_NANO_MODEL_CARD,
+    Model.o4_MINI.value: o4_MINI_MODEL_CARD,
     # Model.o1.value: o1_MODEL_CARD,
     Model.TEXT_EMBEDDING_3_SMALL.value: TEXT_EMBEDDING_3_SMALL_MODEL_CARD,
     Model.CLIP_VIT_B_32.value: CLIP_VIT_B_32_MODEL_CARD,
+    Model.CLAUDE_3_5_SONNET.value: CLAUDE_3_5_SONNET_MODEL_CARD,
+    Model.CLAUDE_3_7_SONNET.value: CLAUDE_3_7_SONNET_MODEL_CARD,
+    Model.CLAUDE_3_5_HAIKU.value: CLAUDE_3_5_HAIKU_MODEL_CARD,
+    Model.GEMINI_2_0_FLASH.value: GEMINI_2_0_FLASH_MODEL_CARD,
+    Model.GEMINI_2_5_FLASH.value: GEMINI_2_5_FLASH_MODEL_CARD,
+    Model.GEMINI_2_5_PRO.value: GEMINI_2_5_PRO_MODEL_CARD,
+    Model.GOOGLE_GEMINI_2_5_FLASH.value: GEMINI_2_5_FLASH_MODEL_CARD,
+    Model.GOOGLE_GEMINI_2_5_FLASH_LITE.value: GEMINI_2_5_FLASH_LITE_MODEL_CARD,
+    Model.GOOGLE_GEMINI_2_5_PRO.value: GEMINI_2_5_PRO_MODEL_CARD,
+    Model.LLAMA_4_MAVERICK.value: LLAMA_4_MAVERICK_MODEL_CARD,
+    Model.VLLM_QWEN_1_5_0_5B_CHAT.value: VLLM_QWEN_1_5_0_5B_CHAT_MODEL_CARD,
 }
-
-
-###### DEPRECATED ######
-# # NOTE: seconds_per_output_token is based on `gpt-3.5-turbo-1106`
-# GPT_3_5_MODEL_CARD = {
-#     ##### Cost in USD #####
-#     "usd_per_input_token": 0.5 / 1E6,
-#     "usd_per_output_token": 1.5 / 1E6,
-#     ##### Time #####
-#     "seconds_per_output_token": 0.0065,
-#     ##### Agg. Benchmark #####
-#     "overall": 70.0, # 5-shot
-#     ##### Commonsense Reasoning #####
-#     "reasoning": 84.1,
-#     ### "HellaSwag": 85.5,^  # 10-shot
-#     ### "WinoGrande": 81.6,^ # 5-shot
-#     ### "Arc-e": 85.2,^      # 25-shot
-#     ##### World Knowledge #####
-#     ##### Reading Comprehension #####
-#     ### "DROP": 64.1, # 3-shot
-#     ##### Code #####
-#     "code": 48.1,
-#     ### "HumanEval": 48.1,^ # 0-shot
-#     ##### Math #####
-#     "math": 57.1,
-#     ### "GSM8K": 57.1,^     # 5-shot
-# }
-# # NOTE: the seconds_per_output_token was computed based on a slightly different model ('gpt-4-1106-preview')
-# #       and the benchmark statistics were computed based on the GPT-4 Technical Report; these might be
-# #       slightly innacurate compared to the real numbers for gpt-4-0125-preview, but we'll use them until
-# #       we have something better. (The cost metrics are accurate).
-# GPT_4_MODEL_CARD = {
-#     ##### Cost in USD #####
-#     "usd_per_input_token": 10 / 1E6,
-#     "usd_per_output_token": 30 / 1E6,
-#     ##### Time #####
-#     "seconds_per_output_token": 0.018,
-#     ##### Agg. Benchmark #####
-#     "overall": 86.4, # 5-shot
-#     ##### Commonsense Reasoning #####
-#     "reasoning": 93.0,
-#     ### "HellaSwag": 95.3,^  # 10-shot
-#     ### "WinoGrande": 87.5,^ # 5-shot
-#     ### "Arc-e": 96.3,^      # 25-shot
-#     ##### World Knowledge #####
-#     ##### Reading Comprehension #####
-#     ### "DROP": 80.9, # 3-shot
-#     ##### Code #####
-#     "code": 67.0,
-#     ### "HumanEval": 67.0,^ # 0-shot
-#     ##### Math #####
-#     "math": 92.0,
-#     ### "GSM8K": 92.0,^     # 5-shot
-# }
-
-# # TODO: use cost info in here: https://platform.openai.com/docs/guides/vision/calculating-costs
-# GPT_4V_MODEL_CARD = {
-#     ##### Cost in USD #####
-#     "usd_per_input_token": 10 / 1E6,
-#     "usd_per_output_token": 30 / 1E6,
-#     ##### Time #####
-#     "seconds_per_output_token": 0.042 / 10.0, # TODO: / 10.0 is a hack; need to figure out why time estimates are so off
-#     ##### Agg. Benchmark #####
-#     "overall": 86.4,
-# }
-
-
-# GEMINI_1_MODEL_CARD = {
-#     ##### Cost in USD #####
-#     "usd_per_input_token": 125 / 1E8, # Gemini is free but rate limited for now. Pricing will be updated
-#     "usd_per_output_token": 375 / 1E9,
-#     ##### Time #####
-#     "seconds_per_output_token": 0.042 / 10.0, # TODO:
-#     ##### Agg. Benchmark #####
-#     "overall": 65.0, # 90.0 TODO: we are using the free version of Gemini which is substantially worse than its paid version; I'm manually revising it's quality below that of Mixtral
-#     ##### Commonsense Reasoning #####
-#     "reasoning": 80.0, # 87.8, TODO: see note above on overall
-#     # "HellaSwag": 87.8,  # 10-shot
-#     ##### World Knowledge #####
-#     ##### Reading Comprehension #####
-#     # "DROP": 82.4, # Variable shots ?
-#     ##### Code #####
-#     "code": 74.4,
-#     # "HumanEval": 74.4, # 0-shot (IT)*
-#     # "Natural2Code": 74.9, # 0-shot
-#     ##### Math #####
-#     "math": 94.4,
-#     # "GSM8K": 94.4,     # maj1@32
-#     # "MATH": 53.2,      # 4-shot
-# }
-
-# GEMINI_1V_MODEL_CARD = {
-#     ##### Cost in USD #####
-#     "usd_per_input_token": 25 / 1E6,  # Gemini is free but rate limited for now. Pricing will be updated
-#     "usd_per_output_token": 375 / 1E9,
-#     ##### Time #####
-#     "seconds_per_output_token": 0.042, # / 10.0, # TODO:
-#     ##### Agg. Benchmark #####
-#     "overall": 65.0, # 90.0, TODO: see note above in Gemini_1 model card
-#     ##### Commonsense Reasoning #####
-#     "reasoning": 80.0, # 87.8, TODO: see note above in Gemini_1 model card
-#     # "HellaSwag": 87.8,  # 10-shot
-#     ##### World Knowledge #####
-#     ##### Reading Comprehension #####
-#     # "DROP": 82.4, # Variable shots ?
-#     ##### Code #####
-#     "code": 74.4,
-#     # "HumanEval": 74.4, # 0-shot (IT)*
-#     # "Natural2Code": 74.9, # 0-shot
-#     ##### Math #####
-#     "math": 94.4,
-#     # "GSM8K": 94.4,     # maj1@32
-#     # "MATH": 53.2,      # 4-shot
-# }