edsl 0.1.39.dev2__py3-none-any.whl → 0.1.39.dev3__py3-none-any.whl

edsl/inference_services/GoogleService.py

@@ -1,139 +1,148 @@
-# import os
-from typing import Any, Dict, List, Optional
-import google
-import google.generativeai as genai
-from google.generativeai.types import GenerationConfig
-from google.api_core.exceptions import InvalidArgument
-
-# from edsl.exceptions.general import MissingAPIKeyError
-from edsl.language_models.LanguageModel import LanguageModel
-from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
-from edsl.coop import Coop
-
-safety_settings = [
-    {
-        "category": "HARM_CATEGORY_HARASSMENT",
-        "threshold": "BLOCK_NONE",
-    },
-    {
-        "category": "HARM_CATEGORY_HATE_SPEECH",
-        "threshold": "BLOCK_NONE",
-    },
-    {
-        "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
-        "threshold": "BLOCK_NONE",
-    },
-    {
-        "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
-        "threshold": "BLOCK_NONE",
-    },
-]
-
-
-class GoogleService(InferenceServiceABC):
-    _inference_service_ = "google"
-    key_sequence = ["candidates", 0, "content", "parts", 0, "text"]
-    usage_sequence = ["usage_metadata"]
-    input_token_name = "prompt_token_count"
-    output_token_name = "candidates_token_count"
-
-    model_exclude_list = []
-
-    @classmethod
-    def available(cls) -> List[str]:
-        model_list = []
-        for m in genai.list_models():
-            if "generateContent" in m.supported_generation_methods:
-                model_list.append(m.name.split("/")[-1])
-        return model_list
-
-    @classmethod
-    def create_model(
-        cls, model_name: str = "gemini-pro", model_class_name=None
-    ) -> LanguageModel:
-        if model_class_name is None:
-            model_class_name = cls.to_class_name(model_name)
-
-        class LLM(LanguageModel):
-            _model_ = model_name
-            key_sequence = cls.key_sequence
-            usage_sequence = cls.usage_sequence
-            input_token_name = cls.input_token_name
-            output_token_name = cls.output_token_name
-            _inference_service_ = cls._inference_service_
-
-            _parameters_ = {
-                "temperature": 0.5,
-                "topP": 1,
-                "topK": 1,
-                "maxOutputTokens": 2048,
-                "stopSequences": [],
-            }
-
-            model = None
-
-            def __init__(self, *args, **kwargs):
-                super().__init__(*args, **kwargs)
-
-            def get_generation_config(self) -> GenerationConfig:
-                return GenerationConfig(
-                    temperature=self.temperature,
-                    top_p=self.topP,
-                    top_k=self.topK,
-                    max_output_tokens=self.maxOutputTokens,
-                    stop_sequences=self.stopSequences,
-                )
-
-            async def async_execute_model_call(
-                self,
-                user_prompt: str,
-                system_prompt: str = "",
-                files_list: Optional["Files"] = None,
-            ) -> Dict[str, Any]:
-                generation_config = self.get_generation_config()
-
-                if files_list is None:
-                    files_list = []
-                genai.configure(api_key=self.api_token)
-                if (
-                    system_prompt is not None
-                    and system_prompt != ""
-                    and self._model_ != "gemini-pro"
-                ):
-                    try:
-                        self.generative_model = genai.GenerativeModel(
-                            self._model_,
-                            safety_settings=safety_settings,
-                            system_instruction=system_prompt,
-                        )
-                    except InvalidArgument as e:
-                        print(
-                            f"This model, {self._model_}, does not support system_instruction"
-                        )
-                        print("Will add system_prompt to user_prompt")
-                        user_prompt = f"{system_prompt}\n{user_prompt}"
-                else:
-                    self.generative_model = genai.GenerativeModel(
-                        self._model_,
-                        safety_settings=safety_settings,
-                    )
-                combined_prompt = [user_prompt]
-                for file in files_list:
-                    if "google" not in file.external_locations:
-                        _ = file.upload_google()
-                    gen_ai_file = google.generativeai.types.file_types.File(
-                        file.external_locations["google"]
-                    )
-                    combined_prompt.append(gen_ai_file)
-
-                response = await self.generative_model.generate_content_async(
-                    combined_prompt, generation_config=generation_config
-                )
-                return response.to_dict()
-
-        LLM.__name__ = model_name
-        return LLM
-
-
-if __name__ == "__main__":
-    pass
+import os
+from typing import Any, Dict, List, Optional
+import google
+import google.generativeai as genai
+from google.generativeai.types import GenerationConfig
+from google.api_core.exceptions import InvalidArgument
+
+from edsl.exceptions import MissingAPIKeyError
+from edsl.language_models.LanguageModel import LanguageModel
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.coop import Coop
+
+safety_settings = [
+    {
+        "category": "HARM_CATEGORY_HARASSMENT",
+        "threshold": "BLOCK_NONE",
+    },
+    {
+        "category": "HARM_CATEGORY_HATE_SPEECH",
+        "threshold": "BLOCK_NONE",
+    },
+    {
+        "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+        "threshold": "BLOCK_NONE",
+    },
+    {
+        "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+        "threshold": "BLOCK_NONE",
+    },
+]
+
+
+class GoogleService(InferenceServiceABC):
+    _inference_service_ = "google"
+    key_sequence = ["candidates", 0, "content", "parts", 0, "text"]
+    usage_sequence = ["usage_metadata"]
+    input_token_name = "prompt_token_count"
+    output_token_name = "candidates_token_count"
+
+    model_exclude_list = []
+
+    # @classmethod
+    # def available(cls) -> List[str]:
+    #     return ["gemini-pro", "gemini-1.5-pro", "gemini-1.5-flash", "gemini-1.0-pro"]
+
+    @classmethod
+    def available(cls) -> List[str]:
+        model_list = []
+        for m in genai.list_models():
+            if "generateContent" in m.supported_generation_methods:
+                model_list.append(m.name.split("/")[-1])
+        return model_list
+
+    @classmethod
+    def create_model(
+        cls, model_name: str = "gemini-pro", model_class_name=None
+    ) -> LanguageModel:
+        if model_class_name is None:
+            model_class_name = cls.to_class_name(model_name)
+
+        class LLM(LanguageModel):
+            _model_ = model_name
+            key_sequence = cls.key_sequence
+            usage_sequence = cls.usage_sequence
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
+            _inference_service_ = cls._inference_service_
+
+            _tpm = cls.get_tpm(cls)
+            _rpm = cls.get_rpm(cls)
+
+            _parameters_ = {
+                "temperature": 0.5,
+                "topP": 1,
+                "topK": 1,
+                "maxOutputTokens": 2048,
+                "stopSequences": [],
+            }
+
+            api_token = None
+            model = None
+
+            def __init__(self, *args, **kwargs):
+                super().__init__(*args, **kwargs)
+
+            def get_generation_config(self) -> GenerationConfig:
+                return GenerationConfig(
+                    temperature=self.temperature,
+                    top_p=self.topP,
+                    top_k=self.topK,
+                    max_output_tokens=self.maxOutputTokens,
+                    stop_sequences=self.stopSequences,
+                )
+
+            async def async_execute_model_call(
+                self,
+                user_prompt: str,
+                system_prompt: str = "",
+                files_list: Optional["Files"] = None,
+            ) -> Dict[str, Any]:
+                generation_config = self.get_generation_config()
+
+                if files_list is None:
+                    files_list = []
+
+                genai.configure(api_key=self.api_token)
+                if (
+                    system_prompt is not None
+                    and system_prompt != ""
+                    and self._model_ != "gemini-pro"
+                ):
+                    try:
+                        self.generative_model = genai.GenerativeModel(
+                            self._model_,
+                            safety_settings=safety_settings,
+                            system_instruction=system_prompt,
+                        )
+                    except InvalidArgument as e:
+                        print(
+                            f"This model, {self._model_}, does not support system_instruction"
+                        )
+                        print("Will add system_prompt to user_prompt")
+                        user_prompt = f"{system_prompt}\n{user_prompt}"
+                else:
+                    self.generative_model = genai.GenerativeModel(
+                        self._model_,
+                        safety_settings=safety_settings,
+                    )
+                combined_prompt = [user_prompt]
+                for file in files_list:
+                    if "google" not in file.external_locations:
+                        _ = file.upload_google()
+                    gen_ai_file = google.generativeai.types.file_types.File(
+                        file.external_locations["google"]
+                    )
+                    combined_prompt.append(gen_ai_file)
+
+                response = await self.generative_model.generate_content_async(
+                    combined_prompt, generation_config=generation_config
+                )
+                return response.to_dict()
+
+        LLM.__name__ = model_name
+        return LLM
+
+
+if __name__ == "__main__":
+    pass

edsl/inference_services/GroqService.py

@@ -1,20 +1,20 @@
-from typing import Any, List
-from edsl.inference_services.OpenAIService import OpenAIService
-
-import groq
-
-
-class GroqService(OpenAIService):
-    """DeepInfra service class."""
-
-    _inference_service_ = "groq"
-    _env_key_name_ = "GROQ_API_KEY"
-
-    _sync_client_ = groq.Groq
-    _async_client_ = groq.AsyncGroq
-
-    model_exclude_list = ["whisper-large-v3", "distil-whisper-large-v3-en"]
-
-    # _base_url_ = "https://api.deepinfra.com/v1/openai"
-    _base_url_ = None
-    _models_list_cache: List[str] = []
+from typing import Any, List
+from edsl.inference_services.OpenAIService import OpenAIService
+
+import groq
+
+
+class GroqService(OpenAIService):
+    """DeepInfra service class."""
+
+    _inference_service_ = "groq"
+    _env_key_name_ = "GROQ_API_KEY"
+
+    _sync_client_ = groq.Groq
+    _async_client_ = groq.AsyncGroq
+
+    model_exclude_list = ["whisper-large-v3", "distil-whisper-large-v3-en"]
+
+    # _base_url_ = "https://api.deepinfra.com/v1/openai"
+    _base_url_ = None
+    _models_list_cache: List[str] = []

edsl/inference_services/InferenceServiceABC.py

@@ -1,80 +1,147 @@
-from abc import abstractmethod, ABC
-import re
-from datetime import datetime, timedelta
-from edsl.config import CONFIG
-
-
-class InferenceServiceABC(ABC):
-    """
-    Abstract class for inference services.
-    """
-
-    _coop_config_vars = None
-
-    def __init_subclass__(cls):
-        """
-        Check that the subclass has the required attributes.
-        - `key_sequence` attribute determines...
-        - `model_exclude_list` attribute determines...
-        """
-        must_have_attributes = [
-            "key_sequence",
-            "model_exclude_list",
-            "usage_sequence",
-            "input_token_name",
-            "output_token_name",
-        ]
-        for attr in must_have_attributes:
-            if not hasattr(cls, attr):
-                raise NotImplementedError(
-                    f"Class {cls.__name__} must have a '{attr}' attribute."
-                )
-
-    @property
-    def service_name(self):
-        return self._inference_service_
-
-    @classmethod
-    def _should_refresh_coop_config_vars(cls):
-        """
-        Returns True if config vars have been fetched over 24 hours ago, and False otherwise.
-        """
-
-        if cls._last_config_fetch is None:
-            return True
-        return (datetime.now() - cls._last_config_fetch) > timedelta(hours=24)
-
-    @abstractmethod
-    def available() -> list[str]:
-        """
-        Returns a list of available models for the service.
-        """
-        pass
-
-    @abstractmethod
-    def create_model():
-        """
-        Returns a LanguageModel object.
-        """
-        pass
-
-    @staticmethod
-    def to_class_name(s):
-        """
-        Converts a string to a valid class name.
-
-        >>> InferenceServiceABC.to_class_name("hello world")
-        'HelloWorld'
-        """
-
-        s = re.sub(r"[^a-zA-Z0-9 ]", "", s)
-        s = "".join(word.title() for word in s.split())
-        if s and s[0].isdigit():
-            s = "Class" + s
-        return s
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod()
+from abc import abstractmethod, ABC
+import os
+import re
+from datetime import datetime, timedelta
+from edsl.config import CONFIG
+
+
+class InferenceServiceABC(ABC):
+    """
+    Abstract class for inference services.
+    Anthropic: https://docs.anthropic.com/en/api/rate-limits
+    """
+
+    _coop_config_vars = None
+
+    default_levels = {
+        "google": {"tpm": 2_000_000, "rpm": 15},
+        "openai": {"tpm": 2_000_000, "rpm": 10_000},
+        "anthropic": {"tpm": 2_000_000, "rpm": 500},
+    }
+
+    def __init_subclass__(cls):
+        """
+        Check that the subclass has the required attributes.
+        - `key_sequence` attribute determines...
+        - `model_exclude_list` attribute determines...
+        """
+        if not hasattr(cls, "key_sequence"):
+            raise NotImplementedError(
+                f"Class {cls.__name__} must have a 'key_sequence' attribute."
+            )
+        if not hasattr(cls, "model_exclude_list"):
+            raise NotImplementedError(
+                f"Class {cls.__name__} must have a 'model_exclude_list' attribute."
+            )
+
+    @classmethod
+    def _should_refresh_coop_config_vars(cls):
+        """
+        Returns True if config vars have been fetched over 24 hours ago, and False otherwise.
+        """
+
+        if cls._last_config_fetch is None:
+            return True
+        return (datetime.now() - cls._last_config_fetch) > timedelta(hours=24)
+
+    @classmethod
+    def _get_limt(cls, limit_type: str) -> int:
+        key = f"EDSL_SERVICE_{limit_type.upper()}_{cls._inference_service_.upper()}"
+        if key in os.environ:
+            return int(os.getenv(key))
+
+        if cls._coop_config_vars is None or cls._should_refresh_coop_config_vars():
+            try:
+                from edsl import Coop
+
+                c = Coop()
+                cls._coop_config_vars = c.fetch_rate_limit_config_vars()
+                cls._last_config_fetch = datetime.now()
+                if key in cls._coop_config_vars:
+                    return cls._coop_config_vars[key]
+            except Exception:
+                cls._coop_config_vars = None
+        else:
+            if key in cls._coop_config_vars:
+                return cls._coop_config_vars[key]
+
+        if cls._inference_service_ in cls.default_levels:
+            return int(cls.default_levels[cls._inference_service_][limit_type])
+
+        return int(CONFIG.get(f"EDSL_SERVICE_{limit_type.upper()}_BASELINE"))
+
+    def get_tpm(cls) -> int:
+        """
+        Returns the TPM for the service. If the service is not defined in the environment variables, it will return the baseline TPM.
+        """
+        return cls._get_limt(limit_type="tpm")
+
+    def get_rpm(cls):
+        """
+        Returns the RPM for the service. If the service is not defined in the environment variables, it will return the baseline RPM.
+        """
+        return cls._get_limt(limit_type="rpm")
+
+    @abstractmethod
+    def available() -> list[str]:
+        """
+        Returns a list of available models for the service.
+        """
+        pass
+
+    @abstractmethod
+    def create_model():
+        """
+        Returns a LanguageModel object.
+        """
+        pass
+
+    @staticmethod
+    def to_class_name(s):
+        """
+        Converts a string to a valid class name.
+
+        >>> InferenceServiceABC.to_class_name("hello world")
+        'HelloWorld'
+        """
+
+        s = re.sub(r"[^a-zA-Z0-9 ]", "", s)
+        s = "".join(word.title() for word in s.split())
+        if s and s[0].isdigit():
+            s = "Class" + s
+        return s
+
+
+if __name__ == "__main__":
+    pass
+    # deep_infra_service = DeepInfraService("deep_infra", "DEEP_INFRA_API_KEY")
+    # deep_infra_service.available()
+    # m = deep_infra_service.create_model("microsoft/WizardLM-2-7B")
+    # response = m().hello()
+    # print(response)
+
+    # anthropic_service = AnthropicService("anthropic", "ANTHROPIC_API_KEY")
+    # anthropic_service.available()
+    # m = anthropic_service.create_model("claude-3-opus-20240229")
+    # response = m().hello()
+    # print(response)
+    # factory = OpenAIService("openai", "OPENAI_API")
+    # factory.available()
+    # m = factory.create_model("gpt-3.5-turbo")
+    # response = m().hello()
+
+    # from edsl import QuestionFreeText
+    # results = QuestionFreeText.example().by(m()).run()
+
+    # collection = InferenceServicesCollection([
+    #     OpenAIService,
+    #     AnthropicService,
+    #     DeepInfraService
+    # ])
+
+    # available = collection.available()
+    # factory = collection.create_model_factory(*available[0])
+    # m = factory()
+    # from edsl import QuestionFreeText
+    # results = QuestionFreeText.example().by(m).run()
+    # print(results)