edsl 0.1.39__py3-none-any.whl → 0.1.39.dev1__py3-none-any.whl

edsl/exceptions/__init__.py

@@ -1,54 +1,54 @@
-# from .agents import (
-#     #    AgentAttributeLookupCallbackError,
-#     AgentCombinationError,
-#     #    AgentLacksLLMError,
-#     #    AgentRespondedWithBadJSONError,
-# )
-# from .configuration import (
-#     InvalidEnvironmentVariableError,
-#     MissingEnvironmentVariableError,
-# )
-# from .data import (
-#     DatabaseConnectionError,
-#     DatabaseCRUDError,
-#     DatabaseIntegrityError,
-# )
+from .agents import (
+    #    AgentAttributeLookupCallbackError,
+    AgentCombinationError,
+    #    AgentLacksLLMError,
+    #    AgentRespondedWithBadJSONError,
+)
+from .configuration import (
+    InvalidEnvironmentVariableError,
+    MissingEnvironmentVariableError,
+)
+from .data import (
+    DatabaseConnectionError,
+    DatabaseCRUDError,
+    DatabaseIntegrityError,
+)
 
-# from .scenarios import (
-#     ScenarioError,
-# )
+from .scenarios import (
+    ScenarioError,
+)
 
-# from .general import MissingAPIKeyError
+from .general import MissingAPIKeyError
 
-# from .jobs import JobsRunError, InterviewErrorPriorTaskCanceled, InterviewTimeoutError
+from .jobs import JobsRunError, InterviewErrorPriorTaskCanceled, InterviewTimeoutError
 
-# from .language_models import (
-#     LanguageModelResponseNotJSONError,
-#     LanguageModelMissingAttributeError,
-#     LanguageModelAttributeTypeError,
-#     LanguageModelDoNotAddError,
-# )
-# from .questions import (
-#     QuestionAnswerValidationError,
-#     QuestionAttributeMissing,
-#     QuestionCreationValidationError,
-#     QuestionResponseValidationError,
-#     QuestionSerializationError,
-#     QuestionScenarioRenderError,
-# )
-# from .results import (
-#     ResultsBadMutationstringError,
-#     ResultsColumnNotFoundError,
-#     ResultsInvalidNameError,
-#     ResultsMutateError,
-# )
-# from .surveys import (
-#     SurveyCreationError,
-#     SurveyHasNoRulesError,
-#     SurveyRuleCannotEvaluateError,
-#     SurveyRuleCollectionHasNoRulesAtNodeError,
-#     SurveyRuleReferenceInRuleToUnknownQuestionError,
-#     SurveyRuleRefersToFutureStateError,
-#     SurveyRuleSendsYouBackwardsError,
-#     SurveyRuleSkipLogicSyntaxError,
-# )
+from .language_models import (
+    LanguageModelResponseNotJSONError,
+    LanguageModelMissingAttributeError,
+    LanguageModelAttributeTypeError,
+    LanguageModelDoNotAddError,
+)
+from .questions import (
+    QuestionAnswerValidationError,
+    QuestionAttributeMissing,
+    QuestionCreationValidationError,
+    QuestionResponseValidationError,
+    QuestionSerializationError,
+    QuestionScenarioRenderError,
+)
+from .results import (
+    ResultsBadMutationstringError,
+    ResultsColumnNotFoundError,
+    ResultsInvalidNameError,
+    ResultsMutateError,
+)
+from .surveys import (
+    SurveyCreationError,
+    SurveyHasNoRulesError,
+    SurveyRuleCannotEvaluateError,
+    SurveyRuleCollectionHasNoRulesAtNodeError,
+    SurveyRuleReferenceInRuleToUnknownQuestionError,
+    SurveyRuleRefersToFutureStateError,
+    SurveyRuleSendsYouBackwardsError,
+    SurveyRuleSkipLogicSyntaxError,
+)

edsl/exceptions/agents.py

@@ -1,18 +1,6 @@
 from edsl.exceptions.BaseException import BaseException
 
 
-# from edsl.utilities.utilities import is_notebook
-
-# from IPython.core.error import UsageError
-
-# class AgentListErrorAlternative(UsageError):
-#     def __init__(self, message):
-#         super().__init__(message)
-
-import sys
-from edsl.utilities.is_notebook import is_notebook
-
-
 class AgentListError(BaseException):
     relevant_doc = "https://docs.expectedparrot.com/en/latest/agents.html#agent-lists"
 

edsl/exceptions/questions.py

@@ -1,6 +1,6 @@
 from typing import Any, SupportsIndex
+from jinja2 import Template
 import json
-from pydantic import ValidationError
 
 
 class QuestionErrors(Exception):
@@ -20,35 +20,17 @@ class QuestionAnswerValidationError(QuestionErrors):
     For example, if the question is a multiple choice question, the answer should be drawn from the list of options provided.
     """
 
-    def __init__(
-        self,
-        message="Invalid answer.",
-        pydantic_error: ValidationError = None,
-        data: dict = None,
-        model=None,
-    ):
+    def __init__(self, message="Invalid answer.", data=None, model=None):
         self.message = message
-        self.pydantic_error = pydantic_error
         self.data = data
         self.model = model
         super().__init__(self.message)
 
     def __str__(self):
-        if isinstance(self.message, ValidationError):
-            # If it's a ValidationError, just return the core error message
-            return str(self.message)
-        elif hasattr(self.message, "errors"):
-            # Handle the case where it's already been converted to a string but has errors
-            error_list = self.message.errors()
-            if error_list:
-                return str(error_list[0].get("msg", "Unknown error"))
-        return str(self.message)
-
-    # def __str__(self):
-    #     return f"""{repr(self)}
-    #     Data being validated: {self.data}
-    #     Pydnantic Model: {self.model}.
-    #     Reported error: {self.message}."""
+        return f"""{repr(self)} 
+        Data being validated: {self.data} 
+        Pydnantic Model: {self.model}.
+        Reported error: {self.message}."""
 
     def to_html_dict(self):
         return {

edsl/exceptions/scenarios.py

@@ -1,13 +1,6 @@
 import re
 import textwrap
 
-# from IPython.core.error import UsageError
-
-
-class AgentListError(Exception):
-    def __init__(self, message):
-        super().__init__(message)
-
 
 class ScenarioError(Exception):
     documentation = "https://docs.expectedparrot.com/en/latest/scenarios.html#module-edsl.scenarios.Scenario"

edsl/inference_services/AnthropicService.py

@@ -11,27 +11,21 @@ class AnthropicService(InferenceServiceABC):
 
     _inference_service_ = "anthropic"
     _env_key_name_ = "ANTHROPIC_API_KEY"
-    key_sequence = ["content", 0, "text"]
+    key_sequence = ["content", 0, "text"]  # ["content"][0]["text"]
     usage_sequence = ["usage"]
     input_token_name = "input_tokens"
     output_token_name = "output_tokens"
     model_exclude_list = []
 
-    @classmethod
-    def get_model_list(cls, api_key: str = None):
-
-        import requests
-
-        if api_key is None:
-            api_key = os.environ.get("ANTHROPIC_API_KEY")
-        headers = {"x-api-key": api_key, "anthropic-version": "2023-06-01"}
-        response = requests.get("https://api.anthropic.com/v1/models", headers=headers)
-        model_names = [m["id"] for m in response.json()["data"]]
-        return model_names
-
     @classmethod
     def available(cls):
-        return cls.get_model_list()
+        # TODO - replace with an API call
+        return [
+            "claude-3-5-sonnet-20240620",
+            "claude-3-opus-20240229",
+            "claude-3-sonnet-20240229",
+            "claude-3-haiku-20240307",
+        ]
 
     @classmethod
     def create_model(
@@ -62,42 +56,29 @@ class AnthropicService(InferenceServiceABC):
                 "top_logprobs": 3,
             }
 
+            _tpm = cls.get_tpm(cls)
+            _rpm = cls.get_rpm(cls)
+
             async def async_execute_model_call(
                 self,
                 user_prompt: str,
                 system_prompt: str = "",
                 files_list: Optional[List["Files"]] = None,
             ) -> dict[str, Any]:
-                """Calls the Anthropic API and returns the API response."""
+                """Calls the OpenAI API and returns the API response."""
 
-                messages = [
-                    {
-                        "role": "user",
-                        "content": [{"type": "text", "text": user_prompt}],
-                    }
-                ]
-                if files_list:
-                    for file_entry in files_list:
-                        encoded_image = file_entry.base64_string
-                        messages[0]["content"].append(
-                            {
-                                "type": "image",
-                                "source": {
-                                    "type": "base64",
-                                    "media_type": file_entry.mime_type,
-                                    "data": encoded_image,
-                                },
-                            }
-                        )
-                # breakpoint()
-                client = AsyncAnthropic(api_key=self.api_token)
+                api_key = os.environ.get("ANTHROPIC_API_KEY")
+                client = AsyncAnthropic(api_key=api_key)
 
                 response = await client.messages.create(
                     model=model_name,
                     max_tokens=self.max_tokens,
                     temperature=self.temperature,
-                    system=system_prompt,  # note that the Anthropic API uses "system" parameter rather than put it in the message
-                    messages=messages,
+                    system=system_prompt,
+                    messages=[
+                        #                        {"role": "system", "content": system_prompt},
+                        {"role": "user", "content": user_prompt},
+                    ],
                 )
                 return response.model_dump()
 

edsl/inference_services/AwsBedrock.py

@@ -69,6 +69,8 @@ class AwsBedrockService(InferenceServiceABC):
             }
             input_token_name = cls.input_token_name
             output_token_name = cls.output_token_name
+            _rpm = cls.get_rpm(cls)
+            _tpm = cls.get_tpm(cls)
 
             async def async_execute_model_call(
                 self,

edsl/inference_services/AzureAI.py

@@ -118,6 +118,8 @@ class AzureAIService(InferenceServiceABC):
                 "max_tokens": 512,
                 "top_p": 0.9,
             }
+            _rpm = cls.get_rpm(cls)
+            _tpm = cls.get_tpm(cls)
 
             async def async_execute_model_call(
                 self,

edsl/inference_services/GoogleService.py

@@ -1,11 +1,11 @@
-# import os
+import os
 from typing import Any, Dict, List, Optional
 import google
 import google.generativeai as genai
 from google.generativeai.types import GenerationConfig
 from google.api_core.exceptions import InvalidArgument
 
-# from edsl.exceptions.general import MissingAPIKeyError
+from edsl.exceptions import MissingAPIKeyError
 from edsl.language_models.LanguageModel import LanguageModel
 from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
 from edsl.coop import Coop
@@ -39,18 +39,18 @@ class GoogleService(InferenceServiceABC):
 
     model_exclude_list = []
 
+    # @classmethod
+    # def available(cls) -> List[str]:
+    #     return ["gemini-pro", "gemini-1.5-pro", "gemini-1.5-flash", "gemini-1.0-pro"]
+
     @classmethod
-    def get_model_list(cls):
+    def available(cls) -> List[str]:
         model_list = []
         for m in genai.list_models():
             if "generateContent" in m.supported_generation_methods:
                 model_list.append(m.name.split("/")[-1])
         return model_list
 
-    @classmethod
-    def available(cls) -> List[str]:
-        return cls.get_model_list()
-
     @classmethod
     def create_model(
         cls, model_name: str = "gemini-pro", model_class_name=None
@@ -66,6 +66,9 @@ class GoogleService(InferenceServiceABC):
             output_token_name = cls.output_token_name
             _inference_service_ = cls._inference_service_
 
+            _tpm = cls.get_tpm(cls)
+            _rpm = cls.get_rpm(cls)
+
             _parameters_ = {
                 "temperature": 0.5,
                 "topP": 1,
@@ -74,6 +77,7 @@ class GoogleService(InferenceServiceABC):
                 "stopSequences": [],
             }
 
+            api_token = None
             model = None
 
             def __init__(self, *args, **kwargs):
@@ -98,6 +102,7 @@ class GoogleService(InferenceServiceABC):
 
                 if files_list is None:
                     files_list = []
+
                 genai.configure(api_key=self.api_token)
                 if (
                     system_prompt is not None

edsl/inference_services/InferenceServiceABC.py

@@ -1,4 +1,5 @@
 from abc import abstractmethod, ABC
+import os
 import re
 from datetime import datetime, timedelta
 from edsl.config import CONFIG
@@ -7,32 +8,31 @@ from edsl.config import CONFIG
 class InferenceServiceABC(ABC):
     """
     Abstract class for inference services.
+    Anthropic: https://docs.anthropic.com/en/api/rate-limits
     """
 
     _coop_config_vars = None
 
+    default_levels = {
+        "google": {"tpm": 2_000_000, "rpm": 15},
+        "openai": {"tpm": 2_000_000, "rpm": 10_000},
+        "anthropic": {"tpm": 2_000_000, "rpm": 500},
+    }
+
     def __init_subclass__(cls):
         """
         Check that the subclass has the required attributes.
         - `key_sequence` attribute determines...
         - `model_exclude_list` attribute determines...
         """
-        must_have_attributes = [
-            "key_sequence",
-            "model_exclude_list",
-            "usage_sequence",
-            "input_token_name",
-            "output_token_name",
-        ]
-        for attr in must_have_attributes:
-            if not hasattr(cls, attr):
-                raise NotImplementedError(
-                    f"Class {cls.__name__} must have a '{attr}' attribute."
-                )
-
-    @property
-    def service_name(self):
-        return self._inference_service_
+        if not hasattr(cls, "key_sequence"):
+            raise NotImplementedError(
+                f"Class {cls.__name__} must have a 'key_sequence' attribute."
+            )
+        if not hasattr(cls, "model_exclude_list"):
+            raise NotImplementedError(
+                f"Class {cls.__name__} must have a 'model_exclude_list' attribute."
+            )
 
     @classmethod
     def _should_refresh_coop_config_vars(cls):
@@ -44,6 +44,44 @@ class InferenceServiceABC(ABC):
             return True
         return (datetime.now() - cls._last_config_fetch) > timedelta(hours=24)
 
+    @classmethod
+    def _get_limt(cls, limit_type: str) -> int:
+        key = f"EDSL_SERVICE_{limit_type.upper()}_{cls._inference_service_.upper()}"
+        if key in os.environ:
+            return int(os.getenv(key))
+
+        if cls._coop_config_vars is None or cls._should_refresh_coop_config_vars():
+            try:
+                from edsl import Coop
+
+                c = Coop()
+                cls._coop_config_vars = c.fetch_rate_limit_config_vars()
+                cls._last_config_fetch = datetime.now()
+                if key in cls._coop_config_vars:
+                    return cls._coop_config_vars[key]
+            except Exception:
+                cls._coop_config_vars = None
+        else:
+            if key in cls._coop_config_vars:
+                return cls._coop_config_vars[key]
+
+        if cls._inference_service_ in cls.default_levels:
+            return int(cls.default_levels[cls._inference_service_][limit_type])
+
+        return int(CONFIG.get(f"EDSL_SERVICE_{limit_type.upper()}_BASELINE"))
+
+    def get_tpm(cls) -> int:
+        """
+        Returns the TPM for the service. If the service is not defined in the environment variables, it will return the baseline TPM.
+        """
+        return cls._get_limt(limit_type="tpm")
+
+    def get_rpm(cls):
+        """
+        Returns the RPM for the service. If the service is not defined in the environment variables, it will return the baseline RPM.
+        """
+        return cls._get_limt(limit_type="rpm")
+
     @abstractmethod
     def available() -> list[str]:
         """
@@ -75,6 +113,35 @@ class InferenceServiceABC(ABC):
 
 
 if __name__ == "__main__":
-    import doctest
+    pass
+    # deep_infra_service = DeepInfraService("deep_infra", "DEEP_INFRA_API_KEY")
+    # deep_infra_service.available()
+    # m = deep_infra_service.create_model("microsoft/WizardLM-2-7B")
+    # response = m().hello()
+    # print(response)
+
+    # anthropic_service = AnthropicService("anthropic", "ANTHROPIC_API_KEY")
+    # anthropic_service.available()
+    # m = anthropic_service.create_model("claude-3-opus-20240229")
+    # response = m().hello()
+    # print(response)
+    # factory = OpenAIService("openai", "OPENAI_API")
+    # factory.available()
+    # m = factory.create_model("gpt-3.5-turbo")
+    # response = m().hello()
+
+    # from edsl import QuestionFreeText
+    # results = QuestionFreeText.example().by(m()).run()
+
+    # collection = InferenceServicesCollection([
+    #     OpenAIService,
+    #     AnthropicService,
+    #     DeepInfraService
+    # ])
 
-    doctest.testmod()
+    # available = collection.available()
+    # factory = collection.create_model_factory(*available[0])
+    # m = factory()
+    # from edsl import QuestionFreeText
+    # results = QuestionFreeText.example().by(m).run()
+    # print(results)