edsl 0.1.38.dev2__py3-none-any.whl → 0.1.38.dev3__py3-none-any.whl

edsl/inference_services/TogetherAIService.py

@@ -1,170 +1,170 @@
-import aiohttp
-import json
-import requests
-from typing import Any, List, Optional
-
-# from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
-from edsl.language_models import LanguageModel
-
-from edsl.inference_services.OpenAIService import OpenAIService
-import openai
-
-
-class TogetherAIService(OpenAIService):
-    """DeepInfra service class."""
-
-    _inference_service_ = "together"
-    _env_key_name_ = "TOGETHER_API_KEY"
-    _base_url_ = "https://api.together.xyz/v1"
-    _models_list_cache: List[str] = []
-
-    # These are non-serverless models. There was no api param to filter them
-    model_exclude_list = [
-        "EleutherAI/llemma_7b",
-        "HuggingFaceH4/zephyr-7b-beta",
-        "Nexusflow/NexusRaven-V2-13B",
-        "NousResearch/Hermes-2-Theta-Llama-3-70B",
-        "NousResearch/Nous-Capybara-7B-V1p9",
-        "NousResearch/Nous-Hermes-13b",
-        "NousResearch/Nous-Hermes-2-Mistral-7B-DPO",
-        "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT",
-        "NousResearch/Nous-Hermes-Llama2-13b",
-        "NousResearch/Nous-Hermes-Llama2-70b",
-        "NousResearch/Nous-Hermes-llama-2-7b",
-        "NumbersStation/nsql-llama-2-7B",
-        "Open-Orca/Mistral-7B-OpenOrca",
-        "Phind/Phind-CodeLlama-34B-Python-v1",
-        "Phind/Phind-CodeLlama-34B-v2",
-        "Qwen/Qwen1.5-0.5B",
-        "Qwen/Qwen1.5-0.5B-Chat",
-        "Qwen/Qwen1.5-1.8B",
-        "Qwen/Qwen1.5-1.8B-Chat",
-        "Qwen/Qwen1.5-14B",
-        "Qwen/Qwen1.5-14B-Chat",
-        "Qwen/Qwen1.5-32B",
-        "Qwen/Qwen1.5-32B-Chat",
-        "Qwen/Qwen1.5-4B",
-        "Qwen/Qwen1.5-4B-Chat",
-        "Qwen/Qwen1.5-72B",
-        "Qwen/Qwen1.5-7B",
-        "Qwen/Qwen1.5-7B-Chat",
-        "Qwen/Qwen2-1.5B",
-        "Qwen/Qwen2-1.5B-Instruct",
-        "Qwen/Qwen2-72B",
-        "Qwen/Qwen2-7B",
-        "Qwen/Qwen2-7B-Instruct",
-        "SG161222/Realistic_Vision_V3.0_VAE",
-        "Snowflake/snowflake-arctic-instruct",
-        "Undi95/ReMM-SLERP-L2-13B",
-        "Undi95/Toppy-M-7B",
-        "WizardLM/WizardCoder-Python-34B-V1.0",
-        "WizardLM/WizardLM-13B-V1.2",
-        "WizardLM/WizardLM-70B-V1.0",
-        "allenai/OLMo-7B",
-        "allenai/OLMo-7B-Instruct",
-        "bert-base-uncased",
-        "codellama/CodeLlama-13b-Instruct-hf",
-        "codellama/CodeLlama-13b-Python-hf",
-        "codellama/CodeLlama-13b-hf",
-        "codellama/CodeLlama-34b-Python-hf",
-        "codellama/CodeLlama-34b-hf",
-        "codellama/CodeLlama-70b-Instruct-hf",
-        "codellama/CodeLlama-70b-Python-hf",
-        "codellama/CodeLlama-70b-hf",
-        "codellama/CodeLlama-7b-Instruct-hf",
-        "codellama/CodeLlama-7b-Python-hf",
-        "codellama/CodeLlama-7b-hf",
-        "cognitivecomputations/dolphin-2.5-mixtral-8x7b",
-        "deepseek-ai/deepseek-coder-33b-instruct",
-        "garage-bAInd/Platypus2-70B-instruct",
-        "google/gemma-2b",
-        "google/gemma-7b",
-        "google/gemma-7b-it",
-        "gradientai/Llama-3-70B-Instruct-Gradient-1048k",
-        "hazyresearch/M2-BERT-2k-Retrieval-Encoder-V1",
-        "huggyllama/llama-13b",
-        "huggyllama/llama-30b",
-        "huggyllama/llama-65b",
-        "huggyllama/llama-7b",
-        "lmsys/vicuna-13b-v1.3",
-        "lmsys/vicuna-13b-v1.5",
-        "lmsys/vicuna-13b-v1.5-16k",
-        "lmsys/vicuna-7b-v1.3",
-        "lmsys/vicuna-7b-v1.5",
-        "meta-llama/Llama-2-13b-hf",
-        "meta-llama/Llama-2-70b-chat-hf",
-        "meta-llama/Llama-2-7b-hf",
-        "meta-llama/Llama-3-70b-hf",
-        "meta-llama/Llama-3-8b-hf",
-        "meta-llama/Meta-Llama-3-70B",
-        "meta-llama/Meta-Llama-3-70B-Instruct",
-        "meta-llama/Meta-Llama-3-8B-Instruct",
-        "meta-llama/Meta-Llama-3.1-70B-Instruct-Reference",
-        "meta-llama/Meta-Llama-3.1-70B-Reference",
-        "meta-llama/Meta-Llama-3.1-8B-Reference",
-        "microsoft/phi-2",
-        "mistralai/Mixtral-8x22B",
-        "openchat/openchat-3.5-1210",
-        "prompthero/openjourney",
-        "runwayml/stable-diffusion-v1-5",
-        "sentence-transformers/msmarco-bert-base-dot-v5",
-        "snorkelai/Snorkel-Mistral-PairRM-DPO",
-        "stabilityai/stable-diffusion-2-1",
-        "teknium/OpenHermes-2-Mistral-7B",
-        "teknium/OpenHermes-2p5-Mistral-7B",
-        "togethercomputer/CodeLlama-13b-Instruct",
-        "togethercomputer/CodeLlama-13b-Python",
-        "togethercomputer/CodeLlama-34b",
-        "togethercomputer/CodeLlama-34b-Python",
-        "togethercomputer/CodeLlama-7b-Instruct",
-        "togethercomputer/CodeLlama-7b-Python",
-        "togethercomputer/Koala-13B",
-        "togethercomputer/Koala-7B",
-        "togethercomputer/LLaMA-2-7B-32K",
-        "togethercomputer/SOLAR-10.7B-Instruct-v1.0-int4",
-        "togethercomputer/StripedHyena-Hessian-7B",
-        "togethercomputer/alpaca-7b",
-        "togethercomputer/evo-1-131k-base",
-        "togethercomputer/evo-1-8k-base",
-        "togethercomputer/guanaco-13b",
-        "togethercomputer/guanaco-33b",
-        "togethercomputer/guanaco-65b",
-        "togethercomputer/guanaco-7b",
-        "togethercomputer/llama-2-13b",
-        "togethercomputer/llama-2-70b-chat",
-        "togethercomputer/llama-2-7b",
-        "wavymulder/Analog-Diffusion",
-        "zero-one-ai/Yi-34B",
-        "zero-one-ai/Yi-34B-Chat",
-        "zero-one-ai/Yi-6B",
-    ]
-
-    _sync_client_ = openai.OpenAI
-    _async_client_ = openai.AsyncOpenAI
-
-    @classmethod
-    def get_model_list(cls):
-        # Togheter.ai has a different response in model list then openai
-        # and the OpenAI class returns an error when calling .models.list()
-        import requests
-        import os
-
-        url = "https://api.together.xyz/v1/models?filter=serverless"
-        token = os.getenv(cls._env_key_name_)
-        headers = {"accept": "application/json", "authorization": f"Bearer {token}"}
-
-        response = requests.get(url, headers=headers)
-        return response.json()
-
-    @classmethod
-    def available(cls) -> List[str]:
-        if not cls._models_list_cache:
-            try:
-                cls._models_list_cache = [
-                    m["id"]
-                    for m in cls.get_model_list()
-                    if m["id"] not in cls.model_exclude_list
-                ]
-            except Exception as e:
-                raise
-        return cls._models_list_cache
+import aiohttp
+import json
+import requests
+from typing import Any, List, Optional
+
+# from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models import LanguageModel
+
+from edsl.inference_services.OpenAIService import OpenAIService
+import openai
+
+
+class TogetherAIService(OpenAIService):
+    """DeepInfra service class."""
+
+    _inference_service_ = "together"
+    _env_key_name_ = "TOGETHER_API_KEY"
+    _base_url_ = "https://api.together.xyz/v1"
+    _models_list_cache: List[str] = []
+
+    # These are non-serverless models. There was no api param to filter them
+    model_exclude_list = [
+        "EleutherAI/llemma_7b",
+        "HuggingFaceH4/zephyr-7b-beta",
+        "Nexusflow/NexusRaven-V2-13B",
+        "NousResearch/Hermes-2-Theta-Llama-3-70B",
+        "NousResearch/Nous-Capybara-7B-V1p9",
+        "NousResearch/Nous-Hermes-13b",
+        "NousResearch/Nous-Hermes-2-Mistral-7B-DPO",
+        "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT",
+        "NousResearch/Nous-Hermes-Llama2-13b",
+        "NousResearch/Nous-Hermes-Llama2-70b",
+        "NousResearch/Nous-Hermes-llama-2-7b",
+        "NumbersStation/nsql-llama-2-7B",
+        "Open-Orca/Mistral-7B-OpenOrca",
+        "Phind/Phind-CodeLlama-34B-Python-v1",
+        "Phind/Phind-CodeLlama-34B-v2",
+        "Qwen/Qwen1.5-0.5B",
+        "Qwen/Qwen1.5-0.5B-Chat",
+        "Qwen/Qwen1.5-1.8B",
+        "Qwen/Qwen1.5-1.8B-Chat",
+        "Qwen/Qwen1.5-14B",
+        "Qwen/Qwen1.5-14B-Chat",
+        "Qwen/Qwen1.5-32B",
+        "Qwen/Qwen1.5-32B-Chat",
+        "Qwen/Qwen1.5-4B",
+        "Qwen/Qwen1.5-4B-Chat",
+        "Qwen/Qwen1.5-72B",
+        "Qwen/Qwen1.5-7B",
+        "Qwen/Qwen1.5-7B-Chat",
+        "Qwen/Qwen2-1.5B",
+        "Qwen/Qwen2-1.5B-Instruct",
+        "Qwen/Qwen2-72B",
+        "Qwen/Qwen2-7B",
+        "Qwen/Qwen2-7B-Instruct",
+        "SG161222/Realistic_Vision_V3.0_VAE",
+        "Snowflake/snowflake-arctic-instruct",
+        "Undi95/ReMM-SLERP-L2-13B",
+        "Undi95/Toppy-M-7B",
+        "WizardLM/WizardCoder-Python-34B-V1.0",
+        "WizardLM/WizardLM-13B-V1.2",
+        "WizardLM/WizardLM-70B-V1.0",
+        "allenai/OLMo-7B",
+        "allenai/OLMo-7B-Instruct",
+        "bert-base-uncased",
+        "codellama/CodeLlama-13b-Instruct-hf",
+        "codellama/CodeLlama-13b-Python-hf",
+        "codellama/CodeLlama-13b-hf",
+        "codellama/CodeLlama-34b-Python-hf",
+        "codellama/CodeLlama-34b-hf",
+        "codellama/CodeLlama-70b-Instruct-hf",
+        "codellama/CodeLlama-70b-Python-hf",
+        "codellama/CodeLlama-70b-hf",
+        "codellama/CodeLlama-7b-Instruct-hf",
+        "codellama/CodeLlama-7b-Python-hf",
+        "codellama/CodeLlama-7b-hf",
+        "cognitivecomputations/dolphin-2.5-mixtral-8x7b",
+        "deepseek-ai/deepseek-coder-33b-instruct",
+        "garage-bAInd/Platypus2-70B-instruct",
+        "google/gemma-2b",
+        "google/gemma-7b",
+        "google/gemma-7b-it",
+        "gradientai/Llama-3-70B-Instruct-Gradient-1048k",
+        "hazyresearch/M2-BERT-2k-Retrieval-Encoder-V1",
+        "huggyllama/llama-13b",
+        "huggyllama/llama-30b",
+        "huggyllama/llama-65b",
+        "huggyllama/llama-7b",
+        "lmsys/vicuna-13b-v1.3",
+        "lmsys/vicuna-13b-v1.5",
+        "lmsys/vicuna-13b-v1.5-16k",
+        "lmsys/vicuna-7b-v1.3",
+        "lmsys/vicuna-7b-v1.5",
+        "meta-llama/Llama-2-13b-hf",
+        "meta-llama/Llama-2-70b-chat-hf",
+        "meta-llama/Llama-2-7b-hf",
+        "meta-llama/Llama-3-70b-hf",
+        "meta-llama/Llama-3-8b-hf",
+        "meta-llama/Meta-Llama-3-70B",
+        "meta-llama/Meta-Llama-3-70B-Instruct",
+        "meta-llama/Meta-Llama-3-8B-Instruct",
+        "meta-llama/Meta-Llama-3.1-70B-Instruct-Reference",
+        "meta-llama/Meta-Llama-3.1-70B-Reference",
+        "meta-llama/Meta-Llama-3.1-8B-Reference",
+        "microsoft/phi-2",
+        "mistralai/Mixtral-8x22B",
+        "openchat/openchat-3.5-1210",
+        "prompthero/openjourney",
+        "runwayml/stable-diffusion-v1-5",
+        "sentence-transformers/msmarco-bert-base-dot-v5",
+        "snorkelai/Snorkel-Mistral-PairRM-DPO",
+        "stabilityai/stable-diffusion-2-1",
+        "teknium/OpenHermes-2-Mistral-7B",
+        "teknium/OpenHermes-2p5-Mistral-7B",
+        "togethercomputer/CodeLlama-13b-Instruct",
+        "togethercomputer/CodeLlama-13b-Python",
+        "togethercomputer/CodeLlama-34b",
+        "togethercomputer/CodeLlama-34b-Python",
+        "togethercomputer/CodeLlama-7b-Instruct",
+        "togethercomputer/CodeLlama-7b-Python",
+        "togethercomputer/Koala-13B",
+        "togethercomputer/Koala-7B",
+        "togethercomputer/LLaMA-2-7B-32K",
+        "togethercomputer/SOLAR-10.7B-Instruct-v1.0-int4",
+        "togethercomputer/StripedHyena-Hessian-7B",
+        "togethercomputer/alpaca-7b",
+        "togethercomputer/evo-1-131k-base",
+        "togethercomputer/evo-1-8k-base",
+        "togethercomputer/guanaco-13b",
+        "togethercomputer/guanaco-33b",
+        "togethercomputer/guanaco-65b",
+        "togethercomputer/guanaco-7b",
+        "togethercomputer/llama-2-13b",
+        "togethercomputer/llama-2-70b-chat",
+        "togethercomputer/llama-2-7b",
+        "wavymulder/Analog-Diffusion",
+        "zero-one-ai/Yi-34B",
+        "zero-one-ai/Yi-34B-Chat",
+        "zero-one-ai/Yi-6B",
+    ]
+
+    _sync_client_ = openai.OpenAI
+    _async_client_ = openai.AsyncOpenAI
+
+    @classmethod
+    def get_model_list(cls):
+        # Togheter.ai has a different response in model list then openai
+        # and the OpenAI class returns an error when calling .models.list()
+        import requests
+        import os
+
+        url = "https://api.together.xyz/v1/models?filter=serverless"
+        token = os.getenv(cls._env_key_name_)
+        headers = {"accept": "application/json", "authorization": f"Bearer {token}"}
+
+        response = requests.get(url, headers=headers)
+        return response.json()
+
+    @classmethod
+    def available(cls) -> List[str]:
+        if not cls._models_list_cache:
+            try:
+                cls._models_list_cache = [
+                    m["id"]
+                    for m in cls.get_model_list()
+                    if m["id"] not in cls.model_exclude_list
+                ]
+            except Exception as e:
+                raise
+        return cls._models_list_cache

edsl/inference_services/models_available_cache.py

@@ -1,118 +1,118 @@
-models_available = {
-    "openai": [
-        "gpt-3.5-turbo-1106",
-        "gpt-4-0125-preview",
-        "gpt-4-turbo-preview",
-        "gpt-3.5-turbo-16k",
-        "gpt-4-1106-preview",
-        "gpt-4-turbo-2024-04-09",
-        "gpt-3.5-turbo-16k-0613",
-        "gpt-4o-2024-05-13",
-        "gpt-4-turbo",
-        "gpt-3.5-turbo-0613",
-        "gpt-4",
-        "gpt-4-0613",
-        "gpt-3.5-turbo-0125",
-        "gpt-3.5-turbo",
-        "gpt-3.5-turbo-instruct",
-        "gpt-3.5-turbo-instruct-0914",
-        "gpt-3.5-turbo-0301",
-        "gpt-4-vision-preview",
-        "gpt-4-1106-vision-preview",
-        "gpt-4o",
-    ],
-    "anthropic": [
-        "claude-3-5-sonnet-20240620",
-        "claude-3-opus-20240229",
-        "claude-3-sonnet-20240229",
-        "claude-3-haiku-20240307",
-    ],
-    "deep_infra": [
-        "meta-llama/Llama-2-13b-chat-hf",
-        "mistralai/Mixtral-8x22B-Instruct-v0.1",
-        "Gryphe/MythoMax-L2-13b-turbo",
-        "mistralai/Mistral-7B-Instruct-v0.1",
-        "Austism/chronos-hermes-13b-v2",
-        "meta-llama/Llama-2-70b-chat-hf",
-        "mistralai/Mistral-7B-Instruct-v0.3",
-        "meta-llama/Llama-2-7b-chat-hf",
-        "Qwen/Qwen2-72B-Instruct",
-        "HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1",
-        "cognitivecomputations/dolphin-2.6-mixtral-8x7b",
-        "bigcode/starcoder2-15b",
-        "microsoft/WizardLM-2-8x22B",
-        "codellama/CodeLlama-70b-Instruct-hf",
-        "Gryphe/MythoMax-L2-13b",
-        "microsoft/WizardLM-2-7B",
-        "01-ai/Yi-34B-Chat",
-        "bigcode/starcoder2-15b-instruct-v0.1",
-        "mistralai/Mixtral-8x7B-Instruct-v0.1",
-        "openchat/openchat-3.6-8b",
-        "meta-llama/Meta-Llama-3-8B-Instruct",
-        "microsoft/Phi-3-medium-4k-instruct",
-        "Phind/Phind-CodeLlama-34B-v2",
-        "google/codegemma-7b-it",
-        "mistralai/Mistral-7B-Instruct-v0.2",
-        "deepinfra/airoboros-70b",
-        "mistralai/Mixtral-8x22B-v0.1",
-        "llava-hf/llava-1.5-7b-hf",
-        "codellama/CodeLlama-34b-Instruct-hf",
-        "google/gemma-1.1-7b-it",
-        "lizpreciatior/lzlv_70b_fp16_hf",
-        "databricks/dbrx-instruct",
-        "nvidia/Nemotron-4-340B-Instruct",
-        "Qwen/Qwen2-7B-Instruct",
-        "meta-llama/Meta-Llama-3-70B-Instruct",
-        "openchat/openchat_3.5",
-    ],
-    "google": [
-        "gemini-1.0-pro",
-        "gemini-1.0-pro-001",
-        "gemini-1.0-pro-latest",
-        "gemini-1.0-pro-vision-latest",
-        "gemini-1.5-flash",
-        "gemini-1.5-flash-001",
-        "gemini-1.5-flash-001-tuning",
-        "gemini-1.5-flash-002",
-        "gemini-1.5-flash-8b",
-        "gemini-1.5-flash-8b-001",
-        "gemini-1.5-flash-8b-exp-0827",
-        "gemini-1.5-flash-8b-exp-0924",
-        "gemini-1.5-flash-8b-latest",
-        "gemini-1.5-flash-exp-0827",
-        "gemini-1.5-flash-latest",
-        "gemini-1.5-pro",
-        "gemini-1.5-pro-001",
-        "gemini-1.5-pro-002",
-        "gemini-1.5-pro-exp-0801",
-        "gemini-1.5-pro-exp-0827",
-        "gemini-1.5-pro-latest",
-        "gemini-pro",
-        "gemini-pro-vision",
-    ],
-    "bedrock": [
-        "amazon.titan-tg1-large",
-        "amazon.titan-text-lite-v1",
-        "amazon.titan-text-express-v1",
-        "anthropic.claude-instant-v1",
-        "anthropic.claude-v2:1",
-        "anthropic.claude-v2",
-        "anthropic.claude-3-sonnet-20240229-v1:0",
-        "anthropic.claude-3-haiku-20240307-v1:0",
-        "anthropic.claude-3-opus-20240229-v1:0",
-        "anthropic.claude-3-5-sonnet-20240620-v1:0",
-        "cohere.command-text-v14",
-        "cohere.command-r-v1:0",
-        "cohere.command-r-plus-v1:0",
-        "cohere.command-light-text-v14",
-        "meta.llama3-8b-instruct-v1:0",
-        "meta.llama3-70b-instruct-v1:0",
-        "meta.llama3-1-8b-instruct-v1:0",
-        "meta.llama3-1-70b-instruct-v1:0",
-        "meta.llama3-1-405b-instruct-v1:0",
-        "mistral.mistral-7b-instruct-v0:2",
-        "mistral.mixtral-8x7b-instruct-v0:1",
-        "mistral.mistral-large-2402-v1:0",
-        "mistral.mistral-large-2407-v1:0",
-    ],
-}
+models_available = {
+    "openai": [
+        "gpt-3.5-turbo-1106",
+        "gpt-4-0125-preview",
+        "gpt-4-turbo-preview",
+        "gpt-3.5-turbo-16k",
+        "gpt-4-1106-preview",
+        "gpt-4-turbo-2024-04-09",
+        "gpt-3.5-turbo-16k-0613",
+        "gpt-4o-2024-05-13",
+        "gpt-4-turbo",
+        "gpt-3.5-turbo-0613",
+        "gpt-4",
+        "gpt-4-0613",
+        "gpt-3.5-turbo-0125",
+        "gpt-3.5-turbo",
+        "gpt-3.5-turbo-instruct",
+        "gpt-3.5-turbo-instruct-0914",
+        "gpt-3.5-turbo-0301",
+        "gpt-4-vision-preview",
+        "gpt-4-1106-vision-preview",
+        "gpt-4o",
+    ],
+    "anthropic": [
+        "claude-3-5-sonnet-20240620",
+        "claude-3-opus-20240229",
+        "claude-3-sonnet-20240229",
+        "claude-3-haiku-20240307",
+    ],
+    "deep_infra": [
+        "meta-llama/Llama-2-13b-chat-hf",
+        "mistralai/Mixtral-8x22B-Instruct-v0.1",
+        "Gryphe/MythoMax-L2-13b-turbo",
+        "mistralai/Mistral-7B-Instruct-v0.1",
+        "Austism/chronos-hermes-13b-v2",
+        "meta-llama/Llama-2-70b-chat-hf",
+        "mistralai/Mistral-7B-Instruct-v0.3",
+        "meta-llama/Llama-2-7b-chat-hf",
+        "Qwen/Qwen2-72B-Instruct",
+        "HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1",
+        "cognitivecomputations/dolphin-2.6-mixtral-8x7b",
+        "bigcode/starcoder2-15b",
+        "microsoft/WizardLM-2-8x22B",
+        "codellama/CodeLlama-70b-Instruct-hf",
+        "Gryphe/MythoMax-L2-13b",
+        "microsoft/WizardLM-2-7B",
+        "01-ai/Yi-34B-Chat",
+        "bigcode/starcoder2-15b-instruct-v0.1",
+        "mistralai/Mixtral-8x7B-Instruct-v0.1",
+        "openchat/openchat-3.6-8b",
+        "meta-llama/Meta-Llama-3-8B-Instruct",
+        "microsoft/Phi-3-medium-4k-instruct",
+        "Phind/Phind-CodeLlama-34B-v2",
+        "google/codegemma-7b-it",
+        "mistralai/Mistral-7B-Instruct-v0.2",
+        "deepinfra/airoboros-70b",
+        "mistralai/Mixtral-8x22B-v0.1",
+        "llava-hf/llava-1.5-7b-hf",
+        "codellama/CodeLlama-34b-Instruct-hf",
+        "google/gemma-1.1-7b-it",
+        "lizpreciatior/lzlv_70b_fp16_hf",
+        "databricks/dbrx-instruct",
+        "nvidia/Nemotron-4-340B-Instruct",
+        "Qwen/Qwen2-7B-Instruct",
+        "meta-llama/Meta-Llama-3-70B-Instruct",
+        "openchat/openchat_3.5",
+    ],
+    "google": [
+        "gemini-1.0-pro",
+        "gemini-1.0-pro-001",
+        "gemini-1.0-pro-latest",
+        "gemini-1.0-pro-vision-latest",
+        "gemini-1.5-flash",
+        "gemini-1.5-flash-001",
+        "gemini-1.5-flash-001-tuning",
+        "gemini-1.5-flash-002",
+        "gemini-1.5-flash-8b",
+        "gemini-1.5-flash-8b-001",
+        "gemini-1.5-flash-8b-exp-0827",
+        "gemini-1.5-flash-8b-exp-0924",
+        "gemini-1.5-flash-8b-latest",
+        "gemini-1.5-flash-exp-0827",
+        "gemini-1.5-flash-latest",
+        "gemini-1.5-pro",
+        "gemini-1.5-pro-001",
+        "gemini-1.5-pro-002",
+        "gemini-1.5-pro-exp-0801",
+        "gemini-1.5-pro-exp-0827",
+        "gemini-1.5-pro-latest",
+        "gemini-pro",
+        "gemini-pro-vision",
+    ],
+    "bedrock": [
+        "amazon.titan-tg1-large",
+        "amazon.titan-text-lite-v1",
+        "amazon.titan-text-express-v1",
+        "anthropic.claude-instant-v1",
+        "anthropic.claude-v2:1",
+        "anthropic.claude-v2",
+        "anthropic.claude-3-sonnet-20240229-v1:0",
+        "anthropic.claude-3-haiku-20240307-v1:0",
+        "anthropic.claude-3-opus-20240229-v1:0",
+        "anthropic.claude-3-5-sonnet-20240620-v1:0",
+        "cohere.command-text-v14",
+        "cohere.command-r-v1:0",
+        "cohere.command-r-plus-v1:0",
+        "cohere.command-light-text-v14",
+        "meta.llama3-8b-instruct-v1:0",
+        "meta.llama3-70b-instruct-v1:0",
+        "meta.llama3-1-8b-instruct-v1:0",
+        "meta.llama3-1-70b-instruct-v1:0",
+        "meta.llama3-1-405b-instruct-v1:0",
+        "mistral.mistral-7b-instruct-v0:2",
+        "mistral.mixtral-8x7b-instruct-v0:1",
+        "mistral.mistral-large-2402-v1:0",
+        "mistral.mistral-large-2407-v1:0",
+    ],
+}

edsl/inference_services/rate_limits_cache.py

@@ -1,25 +1,25 @@
-rate_limits = {
-    "openai": {
-        "date": "Tue, 02 Jul 2024 15:25:28 GMT",
-        "content-type": "application/json",
-        "transfer-encoding": "chunked",
-        "connection": "keep-alive",
-        "openai-organization": "user-wmu32omw8ulzzutk6mjhtqgk",
-        "openai-processing-ms": "760",
-        "openai-version": "2020-10-01",
-        "strict-transport-security": "max-age=31536000; includeSubDomains",
-        "x-ratelimit-limit-requests": "5000",
-        "x-ratelimit-limit-tokens": "600000",
-        "x-ratelimit-remaining-requests": "4999",
-        "x-ratelimit-remaining-tokens": "599978",
-        "x-ratelimit-reset-requests": "12ms",
-        "x-ratelimit-reset-tokens": "2ms",
-        "x-request-id": "req_971608f3647f660a0cd6537fbe21f69c",
-        "cf-cache-status": "DYNAMIC",
-        "set-cookie": "__cf_bm=MJfUk.0TXdjtiNkUUqlUO2gaN3wzm0iHsRQRWExy52o-1719933928-1.0.1.1-0xk9gFxy_mD1KzAsKQ_HpL2pdQJ90D4B5frt65xU.c9k9QwD0oTBILqXB0rykXNh04Pm1UB1.H_W9sFJVOcSaw; path=/; expires=Tue, 02-Jul-24 15:55:28 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=GbheRct_iw9_I8iLWmt5ZRcLYZ_QVnroCrAt8QMVsUg-1719933928399-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None",
-        "server": "cloudflare",
-        "cf-ray": "89cfa6059bb9b68f-OTP",
-        "content-encoding": "gzip",
-        "alt-svc": 'h3=":443"; ma=86400',
-    }
-}
+rate_limits = {
+    "openai": {
+        "date": "Tue, 02 Jul 2024 15:25:28 GMT",
+        "content-type": "application/json",
+        "transfer-encoding": "chunked",
+        "connection": "keep-alive",
+        "openai-organization": "user-wmu32omw8ulzzutk6mjhtqgk",
+        "openai-processing-ms": "760",
+        "openai-version": "2020-10-01",
+        "strict-transport-security": "max-age=31536000; includeSubDomains",
+        "x-ratelimit-limit-requests": "5000",
+        "x-ratelimit-limit-tokens": "600000",
+        "x-ratelimit-remaining-requests": "4999",
+        "x-ratelimit-remaining-tokens": "599978",
+        "x-ratelimit-reset-requests": "12ms",
+        "x-ratelimit-reset-tokens": "2ms",
+        "x-request-id": "req_971608f3647f660a0cd6537fbe21f69c",
+        "cf-cache-status": "DYNAMIC",
+        "set-cookie": "__cf_bm=MJfUk.0TXdjtiNkUUqlUO2gaN3wzm0iHsRQRWExy52o-1719933928-1.0.1.1-0xk9gFxy_mD1KzAsKQ_HpL2pdQJ90D4B5frt65xU.c9k9QwD0oTBILqXB0rykXNh04Pm1UB1.H_W9sFJVOcSaw; path=/; expires=Tue, 02-Jul-24 15:55:28 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=GbheRct_iw9_I8iLWmt5ZRcLYZ_QVnroCrAt8QMVsUg-1719933928399-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None",
+        "server": "cloudflare",
+        "cf-ray": "89cfa6059bb9b68f-OTP",
+        "content-encoding": "gzip",
+        "alt-svc": 'h3=":443"; ma=86400',
+    }
+}