edsl 0.1.38.dev3__py3-none-any.whl → 0.1.39__py3-none-any.whl

edsl/inference_services/ServiceAvailability.py

@@ -0,0 +1,135 @@
+from enum import Enum
+from typing import List, Optional, TYPE_CHECKING
+from functools import partial
+import warnings
+
+from edsl.inference_services.data_structures import AvailableModels, ModelNamesList
+
+if TYPE_CHECKING:
+    from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+
+
+class ModelSource(Enum):
+    LOCAL = "local"
+    COOP = "coop"
+    CACHE = "cache"
+
+
+class ServiceAvailability:
+    """This class is responsible for fetching the available models from different sources."""
+
+    _coop_model_list = None
+
+    def __init__(self, source_order: Optional[List[ModelSource]] = None):
+        """
+        Initialize with custom source order.
+        Default order is LOCAL -> COOP -> CACHE
+        """
+        self.source_order = source_order or [
+            ModelSource.LOCAL,
+            ModelSource.COOP,
+            ModelSource.CACHE,
+        ]
+
+        # Map sources to their fetch functions
+        self._source_fetchers = {
+            ModelSource.LOCAL: self._fetch_from_local_service,
+            ModelSource.COOP: self._fetch_from_coop,
+            ModelSource.CACHE: self._fetch_from_cache,
+        }
+
+    @classmethod
+    def models_from_coop(cls) -> AvailableModels:
+        if not cls._coop_model_list:
+            from edsl.coop.coop import Coop
+
+            c = Coop()
+            coop_model_list = c.fetch_models()
+            cls._coop_model_list = coop_model_list
+        return cls._coop_model_list
+
+    def get_service_available(
+        self, service: "InferenceServiceABC", warn: bool = False
+    ) -> ModelNamesList:
+        """
+        Try to fetch available models from sources in specified order.
+        Returns first successful result.
+        """
+        last_error = None
+
+        for source in self.source_order:
+            try:
+                fetch_func = partial(self._source_fetchers[source], service)
+                result = fetch_func()
+
+                # Cache successful result
+                service._models_list_cache = result
+                return result
+
+            except Exception as e:
+                last_error = e
+                if warn:
+                    self._warn_source_failed(service, source)
+                continue
+
+        # If we get here, all sources failed
+        raise RuntimeError(
+            f"All sources failed to fetch models. Last error: {last_error}"
+        )
+
+    @staticmethod
+    def _fetch_from_local_service(service: "InferenceServiceABC") -> ModelNamesList:
+        """Attempt to fetch models directly from the service."""
+        return service.available()
+
+    @classmethod
+    def _fetch_from_coop(cls, service: "InferenceServiceABC") -> ModelNamesList:
+        """Fetch models from Coop."""
+        models_from_coop = cls.models_from_coop()
+        return models_from_coop.get(service._inference_service_, [])
+
+    @staticmethod
+    def _fetch_from_cache(service: "InferenceServiceABC") -> ModelNamesList:
+        """Fetch models from local cache."""
+        from edsl.inference_services.models_available_cache import models_available
+
+        return models_available.get(service._inference_service_, [])
+
+    def _warn_source_failed(self, service: "InferenceServiceABC", source: ModelSource):
+        """Display appropriate warning message based on failed source."""
+        messages = {
+            ModelSource.LOCAL: f"""Error getting models for {service._inference_service_}. 
+                Check that you have properly stored your Expected Parrot API key and activated remote inference, 
+                or stored your own API keys for the language models that you want to use.
+                See https://docs.expectedparrot.com/en/latest/api_keys.html for instructions on storing API keys.
+                Trying next source.""",
+            ModelSource.COOP: f"Error getting models from Coop for {service._inference_service_}. Trying next source.",
+            ModelSource.CACHE: f"Error getting models from cache for {service._inference_service_}.",
+        }
+        warnings.warn(messages[source], UserWarning)
+
+
+if __name__ == "__main__":
+    # sa = ServiceAvailability()
+    # models_from_coop = sa.models_from_coop()
+    # print(models_from_coop)
+    from edsl.inference_services.OpenAIService import OpenAIService
+
+    openai_models = ServiceAvailability._fetch_from_local_service(OpenAIService())
+    print(openai_models)
+
+# Example usage:
+"""
+# Default order (LOCAL -> COOP -> CACHE)
+availability = ServiceAvailability()
+
+# Custom order (COOP -> LOCAL -> CACHE)
+availability_coop_first = ServiceAvailability([
+    ModelSource.COOP,
+    ModelSource.LOCAL,
+    ModelSource.CACHE
+])
+
+# Get available models using custom order
+models = availability_coop_first.get_service_available(service, warn=True)
+"""

edsl/inference_services/TestService.py

@@ -1,89 +1,90 @@
-from typing import Any, List, Optional
-import os
-import asyncio
-from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
-from edsl.language_models import LanguageModel
-from edsl.inference_services.rate_limits_cache import rate_limits
-from edsl.utilities.utilities import fix_partial_correct_response
-
-from edsl.enums import InferenceServiceType
-import random
-
-
-class TestService(InferenceServiceABC):
-    """OpenAI service class."""
-
-    _inference_service_ = "test"
-    _env_key_name_ = None
-    _base_url_ = None
-
-    _sync_client_ = None
-    _async_client_ = None
-
-    _sync_client_instance = None
-    _async_client_instance = None
-
-    key_sequence = None
-    usage_sequence = None
-    model_exclude_list = []
-    input_token_name = "prompt_tokens"
-    output_token_name = "completion_tokens"
-
-    @classmethod
-    def available(cls) -> list[str]:
-        return ["test"]
-
-    @classmethod
-    def create_model(cls, model_name, model_class_name=None) -> LanguageModel:
-        throw_exception = False
-
-        class TestServiceLanguageModel(LanguageModel):
-            _model_ = "test"
-            _parameters_ = {"temperature": 0.5}
-            _inference_service_ = InferenceServiceType.TEST.value
-            usage_sequence = ["usage"]
-            key_sequence = ["message", 0, "text"]
-            input_token_name = cls.input_token_name
-            output_token_name = cls.output_token_name
-            _rpm = 1000
-            _tpm = 100000
-
-            @property
-            def _canned_response(self):
-                if hasattr(self, "canned_response"):
-                    return self.canned_response
-                else:
-                    return "Hello, world"
-
-            async def async_execute_model_call(
-                self,
-                user_prompt: str,
-                system_prompt: str,
-                # func: Optional[callable] = None,
-                files_list: Optional[List["File"]] = None,
-            ) -> dict[str, Any]:
-                await asyncio.sleep(0.1)
-                # return {"message": """{"answer": "Hello, world"}"""}
-
-                if hasattr(self, "func"):
-                    return {
-                        "message": [
-                            {"text": self.func(user_prompt, system_prompt, files_list)}
-                        ],
-                        "usage": {"prompt_tokens": 1, "completion_tokens": 1},
-                    }
-
-                if hasattr(self, "throw_exception") and self.throw_exception:
-                    if hasattr(self, "exception_probability"):
-                        p = self.exception_probability
-                    else:
-                        p = 1
-
-                    if random.random() < p:
-                        raise Exception("This is a test error")
-                return {
-                    "message": [{"text": f"{self._canned_response}"}],
-                    "usage": {"prompt_tokens": 1, "completion_tokens": 1},
-                }
-
-        return TestServiceLanguageModel
+from typing import Any, List, Optional
+import os
+import asyncio
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models.LanguageModel import LanguageModel
+from edsl.inference_services.rate_limits_cache import rate_limits
+from edsl.utilities.utilities import fix_partial_correct_response
+
+from edsl.enums import InferenceServiceType
+import random
+
+
+class TestService(InferenceServiceABC):
+    """OpenAI service class."""
+
+    _inference_service_ = "test"
+    _env_key_name_ = None
+    _base_url_ = None
+
+    _sync_client_ = None
+    _async_client_ = None
+
+    _sync_client_instance = None
+    _async_client_instance = None
+
+    key_sequence = None
+    usage_sequence = None
+    model_exclude_list = []
+    input_token_name = "prompt_tokens"
+    output_token_name = "completion_tokens"
+
+    @classmethod
+    def available(cls) -> list[str]:
+        return ["test"]
+
+    @classmethod
+    def create_model(cls, model_name, model_class_name=None) -> LanguageModel:
+        throw_exception = False
+
+        class TestServiceLanguageModel(LanguageModel):
+            _model_ = "test"
+            _parameters_ = {"temperature": 0.5}
+            _inference_service_ = InferenceServiceType.TEST.value
+            usage_sequence = ["usage"]
+            key_sequence = ["message", 0, "text"]
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
+            _rpm = 1000
+            _tpm = 100000
+
+            @property
+            def _canned_response(self):
+                if hasattr(self, "canned_response"):
+
+                    return self.canned_response
+                else:
+                    return "Hello, world"
+
+            async def async_execute_model_call(
+                self,
+                user_prompt: str,
+                system_prompt: str,
+                # func: Optional[callable] = None,
+                files_list: Optional[List["File"]] = None,
+            ) -> dict[str, Any]:
+                await asyncio.sleep(0.1)
+
+                if hasattr(self, "throw_exception") and self.throw_exception:
+                    if hasattr(self, "exception_probability"):
+                        p = self.exception_probability
+                    else:
+                        p = 1
+
+                    if random.random() < p:
+                        raise Exception("This is a test error")
+
+                if hasattr(self, "func"):
+                    return {
+                        "message": [
+                            {"text": self.func(user_prompt, system_prompt, files_list)}
+                        ],
+                        "usage": {"prompt_tokens": 1, "completion_tokens": 1},
+                    }
+
+                return {
+                    "message": [{"text": f"{self._canned_response}"}],
+                    "usage": {"prompt_tokens": 1, "completion_tokens": 1},
+                }
+
+        return TestServiceLanguageModel

edsl/inference_services/TogetherAIService.py

@@ -1,170 +1,172 @@
-import aiohttp
-import json
-import requests
-from typing import Any, List, Optional
-
-# from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
-from edsl.language_models import LanguageModel
-
-from edsl.inference_services.OpenAIService import OpenAIService
-import openai
-
-
-class TogetherAIService(OpenAIService):
-    """DeepInfra service class."""
-
-    _inference_service_ = "together"
-    _env_key_name_ = "TOGETHER_API_KEY"
-    _base_url_ = "https://api.together.xyz/v1"
-    _models_list_cache: List[str] = []
-
-    # These are non-serverless models. There was no api param to filter them
-    model_exclude_list = [
-        "EleutherAI/llemma_7b",
-        "HuggingFaceH4/zephyr-7b-beta",
-        "Nexusflow/NexusRaven-V2-13B",
-        "NousResearch/Hermes-2-Theta-Llama-3-70B",
-        "NousResearch/Nous-Capybara-7B-V1p9",
-        "NousResearch/Nous-Hermes-13b",
-        "NousResearch/Nous-Hermes-2-Mistral-7B-DPO",
-        "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT",
-        "NousResearch/Nous-Hermes-Llama2-13b",
-        "NousResearch/Nous-Hermes-Llama2-70b",
-        "NousResearch/Nous-Hermes-llama-2-7b",
-        "NumbersStation/nsql-llama-2-7B",
-        "Open-Orca/Mistral-7B-OpenOrca",
-        "Phind/Phind-CodeLlama-34B-Python-v1",
-        "Phind/Phind-CodeLlama-34B-v2",
-        "Qwen/Qwen1.5-0.5B",
-        "Qwen/Qwen1.5-0.5B-Chat",
-        "Qwen/Qwen1.5-1.8B",
-        "Qwen/Qwen1.5-1.8B-Chat",
-        "Qwen/Qwen1.5-14B",
-        "Qwen/Qwen1.5-14B-Chat",
-        "Qwen/Qwen1.5-32B",
-        "Qwen/Qwen1.5-32B-Chat",
-        "Qwen/Qwen1.5-4B",
-        "Qwen/Qwen1.5-4B-Chat",
-        "Qwen/Qwen1.5-72B",
-        "Qwen/Qwen1.5-7B",
-        "Qwen/Qwen1.5-7B-Chat",
-        "Qwen/Qwen2-1.5B",
-        "Qwen/Qwen2-1.5B-Instruct",
-        "Qwen/Qwen2-72B",
-        "Qwen/Qwen2-7B",
-        "Qwen/Qwen2-7B-Instruct",
-        "SG161222/Realistic_Vision_V3.0_VAE",
-        "Snowflake/snowflake-arctic-instruct",
-        "Undi95/ReMM-SLERP-L2-13B",
-        "Undi95/Toppy-M-7B",
-        "WizardLM/WizardCoder-Python-34B-V1.0",
-        "WizardLM/WizardLM-13B-V1.2",
-        "WizardLM/WizardLM-70B-V1.0",
-        "allenai/OLMo-7B",
-        "allenai/OLMo-7B-Instruct",
-        "bert-base-uncased",
-        "codellama/CodeLlama-13b-Instruct-hf",
-        "codellama/CodeLlama-13b-Python-hf",
-        "codellama/CodeLlama-13b-hf",
-        "codellama/CodeLlama-34b-Python-hf",
-        "codellama/CodeLlama-34b-hf",
-        "codellama/CodeLlama-70b-Instruct-hf",
-        "codellama/CodeLlama-70b-Python-hf",
-        "codellama/CodeLlama-70b-hf",
-        "codellama/CodeLlama-7b-Instruct-hf",
-        "codellama/CodeLlama-7b-Python-hf",
-        "codellama/CodeLlama-7b-hf",
-        "cognitivecomputations/dolphin-2.5-mixtral-8x7b",
-        "deepseek-ai/deepseek-coder-33b-instruct",
-        "garage-bAInd/Platypus2-70B-instruct",
-        "google/gemma-2b",
-        "google/gemma-7b",
-        "google/gemma-7b-it",
-        "gradientai/Llama-3-70B-Instruct-Gradient-1048k",
-        "hazyresearch/M2-BERT-2k-Retrieval-Encoder-V1",
-        "huggyllama/llama-13b",
-        "huggyllama/llama-30b",
-        "huggyllama/llama-65b",
-        "huggyllama/llama-7b",
-        "lmsys/vicuna-13b-v1.3",
-        "lmsys/vicuna-13b-v1.5",
-        "lmsys/vicuna-13b-v1.5-16k",
-        "lmsys/vicuna-7b-v1.3",
-        "lmsys/vicuna-7b-v1.5",
-        "meta-llama/Llama-2-13b-hf",
-        "meta-llama/Llama-2-70b-chat-hf",
-        "meta-llama/Llama-2-7b-hf",
-        "meta-llama/Llama-3-70b-hf",
-        "meta-llama/Llama-3-8b-hf",
-        "meta-llama/Meta-Llama-3-70B",
-        "meta-llama/Meta-Llama-3-70B-Instruct",
-        "meta-llama/Meta-Llama-3-8B-Instruct",
-        "meta-llama/Meta-Llama-3.1-70B-Instruct-Reference",
-        "meta-llama/Meta-Llama-3.1-70B-Reference",
-        "meta-llama/Meta-Llama-3.1-8B-Reference",
-        "microsoft/phi-2",
-        "mistralai/Mixtral-8x22B",
-        "openchat/openchat-3.5-1210",
-        "prompthero/openjourney",
-        "runwayml/stable-diffusion-v1-5",
-        "sentence-transformers/msmarco-bert-base-dot-v5",
-        "snorkelai/Snorkel-Mistral-PairRM-DPO",
-        "stabilityai/stable-diffusion-2-1",
-        "teknium/OpenHermes-2-Mistral-7B",
-        "teknium/OpenHermes-2p5-Mistral-7B",
-        "togethercomputer/CodeLlama-13b-Instruct",
-        "togethercomputer/CodeLlama-13b-Python",
-        "togethercomputer/CodeLlama-34b",
-        "togethercomputer/CodeLlama-34b-Python",
-        "togethercomputer/CodeLlama-7b-Instruct",
-        "togethercomputer/CodeLlama-7b-Python",
-        "togethercomputer/Koala-13B",
-        "togethercomputer/Koala-7B",
-        "togethercomputer/LLaMA-2-7B-32K",
-        "togethercomputer/SOLAR-10.7B-Instruct-v1.0-int4",
-        "togethercomputer/StripedHyena-Hessian-7B",
-        "togethercomputer/alpaca-7b",
-        "togethercomputer/evo-1-131k-base",
-        "togethercomputer/evo-1-8k-base",
-        "togethercomputer/guanaco-13b",
-        "togethercomputer/guanaco-33b",
-        "togethercomputer/guanaco-65b",
-        "togethercomputer/guanaco-7b",
-        "togethercomputer/llama-2-13b",
-        "togethercomputer/llama-2-70b-chat",
-        "togethercomputer/llama-2-7b",
-        "wavymulder/Analog-Diffusion",
-        "zero-one-ai/Yi-34B",
-        "zero-one-ai/Yi-34B-Chat",
-        "zero-one-ai/Yi-6B",
-    ]
-
-    _sync_client_ = openai.OpenAI
-    _async_client_ = openai.AsyncOpenAI
-
-    @classmethod
-    def get_model_list(cls):
-        # Togheter.ai has a different response in model list then openai
-        # and the OpenAI class returns an error when calling .models.list()
-        import requests
-        import os
-
-        url = "https://api.together.xyz/v1/models?filter=serverless"
-        token = os.getenv(cls._env_key_name_)
-        headers = {"accept": "application/json", "authorization": f"Bearer {token}"}
-
-        response = requests.get(url, headers=headers)
-        return response.json()
-
-    @classmethod
-    def available(cls) -> List[str]:
-        if not cls._models_list_cache:
-            try:
-                cls._models_list_cache = [
-                    m["id"]
-                    for m in cls.get_model_list()
-                    if m["id"] not in cls.model_exclude_list
-                ]
-            except Exception as e:
-                raise
-        return cls._models_list_cache
+import aiohttp
+import json
+import requests
+from typing import Any, List, Optional
+
+# from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models import LanguageModel
+
+from edsl.inference_services.OpenAIService import OpenAIService
+import openai
+
+
+class TogetherAIService(OpenAIService):
+    """DeepInfra service class."""
+
+    _inference_service_ = "together"
+    _env_key_name_ = "TOGETHER_API_KEY"
+    _base_url_ = "https://api.together.xyz/v1"
+    _models_list_cache: List[str] = []
+
+    # These are non-serverless models. There was no api param to filter them
+    model_exclude_list = [
+        "EleutherAI/llemma_7b",
+        "HuggingFaceH4/zephyr-7b-beta",
+        "Nexusflow/NexusRaven-V2-13B",
+        "NousResearch/Hermes-2-Theta-Llama-3-70B",
+        "NousResearch/Nous-Capybara-7B-V1p9",
+        "NousResearch/Nous-Hermes-13b",
+        "NousResearch/Nous-Hermes-2-Mistral-7B-DPO",
+        "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT",
+        "NousResearch/Nous-Hermes-Llama2-13b",
+        "NousResearch/Nous-Hermes-Llama2-70b",
+        "NousResearch/Nous-Hermes-llama-2-7b",
+        "NumbersStation/nsql-llama-2-7B",
+        "Open-Orca/Mistral-7B-OpenOrca",
+        "Phind/Phind-CodeLlama-34B-Python-v1",
+        "Phind/Phind-CodeLlama-34B-v2",
+        "Qwen/Qwen1.5-0.5B",
+        "Qwen/Qwen1.5-0.5B-Chat",
+        "Qwen/Qwen1.5-1.8B",
+        "Qwen/Qwen1.5-1.8B-Chat",
+        "Qwen/Qwen1.5-14B",
+        "Qwen/Qwen1.5-14B-Chat",
+        "Qwen/Qwen1.5-32B",
+        "Qwen/Qwen1.5-32B-Chat",
+        "Qwen/Qwen1.5-4B",
+        "Qwen/Qwen1.5-4B-Chat",
+        "Qwen/Qwen1.5-72B",
+        "Qwen/Qwen1.5-7B",
+        "Qwen/Qwen1.5-7B-Chat",
+        "Qwen/Qwen2-1.5B",
+        "Qwen/Qwen2-1.5B-Instruct",
+        "Qwen/Qwen2-72B",
+        "Qwen/Qwen2-7B",
+        "Qwen/Qwen2-7B-Instruct",
+        "SG161222/Realistic_Vision_V3.0_VAE",
+        "Snowflake/snowflake-arctic-instruct",
+        "Undi95/ReMM-SLERP-L2-13B",
+        "Undi95/Toppy-M-7B",
+        "WizardLM/WizardCoder-Python-34B-V1.0",
+        "WizardLM/WizardLM-13B-V1.2",
+        "WizardLM/WizardLM-70B-V1.0",
+        "allenai/OLMo-7B",
+        "allenai/OLMo-7B-Instruct",
+        "bert-base-uncased",
+        "codellama/CodeLlama-13b-Instruct-hf",
+        "codellama/CodeLlama-13b-Python-hf",
+        "codellama/CodeLlama-13b-hf",
+        "codellama/CodeLlama-34b-Python-hf",
+        "codellama/CodeLlama-34b-hf",
+        "codellama/CodeLlama-70b-Instruct-hf",
+        "codellama/CodeLlama-70b-Python-hf",
+        "codellama/CodeLlama-70b-hf",
+        "codellama/CodeLlama-7b-Instruct-hf",
+        "codellama/CodeLlama-7b-Python-hf",
+        "codellama/CodeLlama-7b-hf",
+        "cognitivecomputations/dolphin-2.5-mixtral-8x7b",
+        "deepseek-ai/deepseek-coder-33b-instruct",
+        "garage-bAInd/Platypus2-70B-instruct",
+        "google/gemma-2b",
+        "google/gemma-7b",
+        "google/gemma-7b-it",
+        "gradientai/Llama-3-70B-Instruct-Gradient-1048k",
+        "hazyresearch/M2-BERT-2k-Retrieval-Encoder-V1",
+        "huggyllama/llama-13b",
+        "huggyllama/llama-30b",
+        "huggyllama/llama-65b",
+        "huggyllama/llama-7b",
+        "lmsys/vicuna-13b-v1.3",
+        "lmsys/vicuna-13b-v1.5",
+        "lmsys/vicuna-13b-v1.5-16k",
+        "lmsys/vicuna-7b-v1.3",
+        "lmsys/vicuna-7b-v1.5",
+        "meta-llama/Llama-2-13b-hf",
+        "meta-llama/Llama-2-70b-chat-hf",
+        "meta-llama/Llama-2-7b-hf",
+        "meta-llama/Llama-3-70b-hf",
+        "meta-llama/Llama-3-8b-hf",
+        "meta-llama/Meta-Llama-3-70B",
+        "meta-llama/Meta-Llama-3-70B-Instruct",
+        "meta-llama/Meta-Llama-3-8B-Instruct",
+        "meta-llama/Meta-Llama-3.1-70B-Instruct-Reference",
+        "meta-llama/Meta-Llama-3.1-70B-Reference",
+        "meta-llama/Meta-Llama-3.1-8B-Reference",
+        "microsoft/phi-2",
+        "mistralai/Mixtral-8x22B",
+        "openchat/openchat-3.5-1210",
+        "prompthero/openjourney",
+        "runwayml/stable-diffusion-v1-5",
+        "sentence-transformers/msmarco-bert-base-dot-v5",
+        "snorkelai/Snorkel-Mistral-PairRM-DPO",
+        "stabilityai/stable-diffusion-2-1",
+        "teknium/OpenHermes-2-Mistral-7B",
+        "teknium/OpenHermes-2p5-Mistral-7B",
+        "togethercomputer/CodeLlama-13b-Instruct",
+        "togethercomputer/CodeLlama-13b-Python",
+        "togethercomputer/CodeLlama-34b",
+        "togethercomputer/CodeLlama-34b-Python",
+        "togethercomputer/CodeLlama-7b-Instruct",
+        "togethercomputer/CodeLlama-7b-Python",
+        "togethercomputer/Koala-13B",
+        "togethercomputer/Koala-7B",
+        "togethercomputer/LLaMA-2-7B-32K",
+        "togethercomputer/SOLAR-10.7B-Instruct-v1.0-int4",
+        "togethercomputer/StripedHyena-Hessian-7B",
+        "togethercomputer/alpaca-7b",
+        "togethercomputer/evo-1-131k-base",
+        "togethercomputer/evo-1-8k-base",
+        "togethercomputer/guanaco-13b",
+        "togethercomputer/guanaco-33b",
+        "togethercomputer/guanaco-65b",
+        "togethercomputer/guanaco-7b",
+        "togethercomputer/llama-2-13b",
+        "togethercomputer/llama-2-70b-chat",
+        "togethercomputer/llama-2-7b",
+        "wavymulder/Analog-Diffusion",
+        "zero-one-ai/Yi-34B",
+        "zero-one-ai/Yi-34B-Chat",
+        "zero-one-ai/Yi-6B",
+    ]
+
+    _sync_client_ = openai.OpenAI
+    _async_client_ = openai.AsyncOpenAI
+
+    @classmethod
+    def get_model_list(cls, api_token=None):
+        # Togheter.ai has a different response in model list then openai
+        # and the OpenAI class returns an error when calling .models.list()
+        import requests
+        import os
+
+        url = "https://api.together.xyz/v1/models?filter=serverless"
+        if api_token is None:
+            api_token = os.getenv(cls._env_key_name_)
+
+        headers = {"accept": "application/json", "authorization": f"Bearer {api_token}"}
+
+        response = requests.get(url, headers=headers)
+        return response.json()
+
+    @classmethod
+    def available(cls) -> List[str]:
+        if not cls._models_list_cache:
+            try:
+                cls._models_list_cache = [
+                    m["id"]
+                    for m in cls.get_model_list()
+                    if m["id"] not in cls.model_exclude_list
+                ]
+            except Exception as e:
+                raise
+        return cls._models_list_cache