edsl 0.1.31.dev4__py3-none-any.whl → 0.1.33__py3-none-any.whl

edsl/exceptions/questions.py

@@ -1,16 +1,73 @@
+from typing import Any, SupportsIndex
+from jinja2 import Template
+import json
+
+
 class QuestionErrors(Exception):
-    pass
+    """
+    Base exception class for question-related errors.
+    """
 
+    def __init__(self, message="An error occurred with the question"):
+        self.message = message
+        super().__init__(self.message)
 
-class QuestionCreationValidationError(QuestionErrors):
-    pass
 
+class QuestionAnswerValidationError(QuestionErrors):
+    documentation = "https://docs.expectedparrot.com/en/latest/exceptions.html"
+
+    explanation = """This when the answer coming from the Language Model does not conform to the expectation for that question type.
+    For example, if the question is a multiple choice question, the answer should be drawn from the list of options provided.
+    """
+
+    def __init__(self, message="Invalid answer.", data=None, model=None):
+        self.message = message
+        self.data = data
+        self.model = model
+        super().__init__(self.message)
+
+    def __str__(self):
+        return f"""{repr(self)} 
+        Data being validated: {self.data} 
+        Pydnantic Model: {self.model}.
+        Reported error: {self.message}."""
+
+    def to_html_dict(self):
+        return {
+            "error_type": ("Name of the exception", "p", "/p", self.__class__.__name__),
+            "explaination": ("Explanation", "p", "/p", self.explanation),
+            "edsl answer": (
+                "What model returned",
+                "pre",
+                "/pre",
+                json.dumps(self.data, indent=2),
+            ),
+            "validating_model": (
+                "Pydantic model for answers",
+                "pre",
+                "/pre",
+                json.dumps(self.model.model_json_schema(), indent=2),
+            ),
+            "error_message": (
+                "Error message Pydantic returned",
+                "p",
+                "/p",
+                self.message,
+            ),
+            "documentation_url": (
+                "URL to EDSL docs",
+                f"a href='{self.documentation}'",
+                "/a",
+                self.documentation,
+            ),
+        }
 
-class QuestionResponseValidationError(QuestionErrors):
+
+class QuestionCreationValidationError(QuestionErrors):
     pass
 
 
-class QuestionAnswerValidationError(QuestionErrors):
+class QuestionResponseValidationError(QuestionErrors):
     pass
 
 

edsl/exceptions/results.py

@@ -2,6 +2,10 @@ class ResultsErrors(Exception):
     pass
 
 
+class ResultsDeserializationError(ResultsErrors):
+    pass
+
+
 class ResultsBadMutationstringError(ResultsErrors):
     pass
 

edsl/inference_services/AnthropicService.py

@@ -11,6 +11,11 @@ class AnthropicService(InferenceServiceABC):
 
     _inference_service_ = "anthropic"
     _env_key_name_ = "ANTHROPIC_API_KEY"
+    key_sequence = ["content", 0, "text"]  # ["content"][0]["text"]
+    usage_sequence = ["usage"]
+    input_token_name = "input_tokens"
+    output_token_name = "output_tokens"
+    model_exclude_list = []
 
     @classmethod
     def available(cls):
@@ -34,6 +39,11 @@ class AnthropicService(InferenceServiceABC):
             Child class of LanguageModel for interacting with OpenAI models
             """
 
+            key_sequence = cls.key_sequence
+            usage_sequence = cls.usage_sequence
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
+
             _inference_service_ = cls._inference_service_
             _model_ = model_name
             _parameters_ = {
@@ -46,6 +56,9 @@ class AnthropicService(InferenceServiceABC):
                 "top_logprobs": 3,
             }
 
+            _tpm = cls.get_tpm(cls)
+            _rpm = cls.get_rpm(cls)
+
             async def async_execute_model_call(
                 self, user_prompt: str, system_prompt: str = ""
             ) -> dict[str, Any]:
@@ -66,17 +79,6 @@ class AnthropicService(InferenceServiceABC):
                 )
                 return response.model_dump()
 
-            @staticmethod
-            def parse_response(raw_response: dict[str, Any]) -> str:
-                """Parses the API response and returns the response text."""
-                response = raw_response["content"][0]["text"]
-                pattern = r"^```json(?:\\n|\n)(.+?)(?:\\n|\n)```$"
-                match = re.match(pattern, response, re.DOTALL)
-                if match:
-                    return match.group(1)
-                else:
-                    return response
-
         LLM.__name__ = model_class_name
 
         return LLM

edsl/inference_services/AwsBedrock.py

@@ -0,0 +1,112 @@
+import os
+from typing import Any
+import re
+import boto3
+from botocore.exceptions import ClientError
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models.LanguageModel import LanguageModel
+import json
+from edsl.utilities.utilities import fix_partial_correct_response
+
+
+class AwsBedrockService(InferenceServiceABC):
+    """AWS Bedrock service class."""
+
+    _inference_service_ = "bedrock"
+    _env_key_name_ = (
+        "AWS_ACCESS_KEY_ID"  # or any other environment key for AWS credentials
+    )
+    key_sequence = ["output", "message", "content", 0, "text"]
+    input_token_name = "inputTokens"
+    output_token_name = "outputTokens"
+    usage_sequence = ["usage"]
+    model_exclude_list = [
+        "ai21.j2-grande-instruct",
+        "ai21.j2-jumbo-instruct",
+        "ai21.j2-mid",
+        "ai21.j2-mid-v1",
+        "ai21.j2-ultra",
+        "ai21.j2-ultra-v1",
+    ]
+
+    @classmethod
+    def available(cls):
+        """Fetch available models from AWS Bedrock."""
+        if not cls._models_list_cache:
+            client = boto3.client("bedrock", region_name="us-west-2")
+            all_models_ids = [
+                x["modelId"] for x in client.list_foundation_models()["modelSummaries"]
+            ]
+        else:
+            all_models_ids = cls._models_list_cache
+
+        return [m for m in all_models_ids if m not in cls.model_exclude_list]
+
+    @classmethod
+    def create_model(
+        cls, model_name: str = "amazon.titan-tg1-large", model_class_name=None
+    ) -> LanguageModel:
+        if model_class_name is None:
+            model_class_name = cls.to_class_name(model_name)
+
+        class LLM(LanguageModel):
+            """
+            Child class of LanguageModel for interacting with AWS Bedrock models.
+            """
+
+            key_sequence = cls.key_sequence
+            usage_sequence = cls.usage_sequence
+            _inference_service_ = cls._inference_service_
+            _model_ = model_name
+            _parameters_ = {
+                "temperature": 0.5,
+                "max_tokens": 512,
+                "top_p": 0.9,
+            }
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
+            _rpm = cls.get_rpm(cls)
+            _tpm = cls.get_tpm(cls)
+
+            async def async_execute_model_call(
+                self, user_prompt: str, system_prompt: str = ""
+            ) -> dict[str, Any]:
+                """Calls the AWS Bedrock API and returns the API response."""
+
+                api_token = (
+                    self.api_token
+                )  # call to check the if env variables are set.
+
+                client = boto3.client("bedrock-runtime", region_name="us-west-2")
+
+                conversation = [
+                    {
+                        "role": "user",
+                        "content": [{"text": user_prompt}],
+                    }
+                ]
+                system = [
+                    {
+                        "text": system_prompt,
+                    }
+                ]
+                try:
+                    response = client.converse(
+                        modelId=self._model_,
+                        messages=conversation,
+                        inferenceConfig={
+                            "maxTokens": self.max_tokens,
+                            "temperature": self.temperature,
+                            "topP": self.top_p,
+                        },
+                        # system=system,
+                        additionalModelRequestFields={},
+                    )
+                    return response
+                except (ClientError, Exception) as e:
+                    print(e)
+                    return {"error": str(e)}
+
+        LLM.__name__ = model_class_name
+
+        return LLM

edsl/inference_services/AzureAI.py

@@ -0,0 +1,214 @@
+import os
+from typing import Any
+import re
+from openai import AsyncAzureOpenAI
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models.LanguageModel import LanguageModel
+
+from azure.ai.inference.aio import ChatCompletionsClient
+from azure.core.credentials import AzureKeyCredential
+from azure.ai.inference.models import SystemMessage, UserMessage
+import asyncio
+import json
+from edsl.utilities.utilities import fix_partial_correct_response
+
+
+def json_handle_none(value: Any) -> Any:
+    """
+    Handle None values during JSON serialization.
+    - Return "null" if the value is None. Otherwise, don't return anything.
+    """
+    if value is None:
+        return "null"
+
+
+class AzureAIService(InferenceServiceABC):
+    """Azure AI service class."""
+
+    # key_sequence = ["content", 0, "text"]  # ["content"][0]["text"]
+    key_sequence = ["choices", 0, "message", "content"]
+    usage_sequence = ["usage"]
+    input_token_name = "prompt_tokens"
+    output_token_name = "completion_tokens"
+
+    _inference_service_ = "azure"
+    _env_key_name_ = (
+        "AZURE_ENDPOINT_URL_AND_KEY"  # Environment variable for Azure API key
+    )
+    _model_id_to_endpoint_and_key = {}
+    model_exclude_list = [
+        "Cohere-command-r-plus-xncmg",
+        "Mistral-Nemo-klfsi",
+        "Mistral-large-2407-ojfld",
+    ]
+
+    @classmethod
+    def available(cls):
+        out = []
+        azure_endpoints = os.getenv("AZURE_ENDPOINT_URL_AND_KEY", None)
+        if not azure_endpoints:
+            raise EnvironmentError(f"AZURE_ENDPOINT_URL_AND_KEY is not defined")
+        azure_endpoints = azure_endpoints.split(",")
+        for data in azure_endpoints:
+            try:
+                # data has this format for non openai models https://model_id.azure_endpoint:azure_key
+                _, endpoint, azure_endpoint_key = data.split(":")
+                if "openai" not in endpoint:
+                    model_id = endpoint.split(".")[0].replace("/", "")
+                    out.append(model_id)
+                    cls._model_id_to_endpoint_and_key[model_id] = {
+                        "endpoint": f"https:{endpoint}",
+                        "azure_endpoint_key": azure_endpoint_key,
+                    }
+                else:
+                    # data has this format for openai models ,https://azure_project_id.openai.azure.com/openai/deployments/gpt-4o-mini/chat/completions?api-version=2023-03-15-preview:azure_key
+                    if "/deployments/" in endpoint:
+                        start_idx = endpoint.index("/deployments/") + len(
+                            "/deployments/"
+                        )
+                        end_idx = (
+                            endpoint.index("/", start_idx)
+                            if "/" in endpoint[start_idx:]
+                            else len(endpoint)
+                        )
+                        model_id = endpoint[start_idx:end_idx]
+                        api_version_value = None
+                        if "api-version=" in endpoint:
+                            start_idx = endpoint.index("api-version=") + len(
+                                "api-version="
+                            )
+                            end_idx = (
+                                endpoint.index("&", start_idx)
+                                if "&" in endpoint[start_idx:]
+                                else len(endpoint)
+                            )
+                            api_version_value = endpoint[start_idx:end_idx]
+
+                        cls._model_id_to_endpoint_and_key[f"azure:{model_id}"] = {
+                            "endpoint": f"https:{endpoint}",
+                            "azure_endpoint_key": azure_endpoint_key,
+                            "api_version": api_version_value,
+                        }
+                        out.append(f"azure:{model_id}")
+
+            except Exception as e:
+                raise e
+        return [m for m in out if m not in cls.model_exclude_list]
+
+    @classmethod
+    def create_model(
+        cls, model_name: str = "azureai", model_class_name=None
+    ) -> LanguageModel:
+        if model_class_name is None:
+            model_class_name = cls.to_class_name(model_name)
+
+        class LLM(LanguageModel):
+            """
+            Child class of LanguageModel for interacting with Azure OpenAI models.
+            """
+
+            key_sequence = cls.key_sequence
+            usage_sequence = cls.usage_sequence
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
+            _inference_service_ = cls._inference_service_
+            _model_ = model_name
+            _parameters_ = {
+                "temperature": 0.5,
+                "max_tokens": 512,
+                "top_p": 0.9,
+            }
+            _rpm = cls.get_rpm(cls)
+            _tpm = cls.get_tpm(cls)
+
+            async def async_execute_model_call(
+                self, user_prompt: str, system_prompt: str = ""
+            ) -> dict[str, Any]:
+                """Calls the Azure OpenAI API and returns the API response."""
+
+                try:
+                    api_key = cls._model_id_to_endpoint_and_key[model_name][
+                        "azure_endpoint_key"
+                    ]
+                except:
+                    api_key = None
+
+                if not api_key:
+                    raise EnvironmentError(
+                        f"AZURE_ENDPOINT_URL_AND_KEY doesn't have the endpoint:key pair for your model: {model_name}"
+                    )
+
+                try:
+                    endpoint = cls._model_id_to_endpoint_and_key[model_name]["endpoint"]
+                except:
+                    endpoint = None
+
+                if not endpoint:
+                    raise EnvironmentError(
+                        f"AZURE_ENDPOINT_URL_AND_KEY doesn't have the endpoint:key pair for your model: {model_name}"
+                    )
+
+                if "openai" not in endpoint:
+                    client = ChatCompletionsClient(
+                        endpoint=endpoint,
+                        credential=AzureKeyCredential(api_key),
+                        temperature=self.temperature,
+                        top_p=self.top_p,
+                        max_tokens=self.max_tokens,
+                    )
+                    try:
+                        response = await client.complete(
+                            messages=[
+                                SystemMessage(content=system_prompt),
+                                UserMessage(content=user_prompt),
+                            ],
+                            # model_extras={"safe_mode": True},
+                        )
+                        await client.close()
+                        return response.as_dict()
+                    except Exception as e:
+                        await client.close()
+                        return {"error": str(e)}
+                else:
+                    api_version = cls._model_id_to_endpoint_and_key[model_name][
+                        "api_version"
+                    ]
+                    client = AsyncAzureOpenAI(
+                        azure_endpoint=endpoint,
+                        api_version=api_version,
+                        api_key=api_key,
+                    )
+                    response = await client.chat.completions.create(
+                        model=model_name,
+                        messages=[
+                            {
+                                "role": "user",
+                                "content": user_prompt,  # Your question can go here
+                            },
+                        ],
+                    )
+                    return response.model_dump()
+
+            # @staticmethod
+            # def parse_response(raw_response: dict[str, Any]) -> str:
+            #     """Parses the API response and returns the response text."""
+            #     if (
+            #         raw_response
+            #         and "choices" in raw_response
+            #         and raw_response["choices"]
+            #     ):
+            #         response = raw_response["choices"][0]["message"]["content"]
+            #         pattern = r"^```json(?:\\n|\n)(.+?)(?:\\n|\n)```$"
+            #         match = re.match(pattern, response, re.DOTALL)
+            #         if match:
+            #             return match.group(1)
+            #         else:
+            #             out = fix_partial_correct_response(response)
+            #             if "error" not in out:
+            #                 response = out["extracted_json"]
+            #             return response
+            #     return "Error parsing response"
+
+        LLM.__name__ = model_class_name
+
+        return LLM

edsl/inference_services/DeepInfraService.py

@@ -2,16 +2,17 @@ import aiohttp
 import json
 import requests
 from typing import Any, List
-#from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+
+# from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
 from edsl.language_models import LanguageModel
 
 from edsl.inference_services.OpenAIService import OpenAIService
 
+
 class DeepInfraService(OpenAIService):
     """DeepInfra service class."""
 
     _inference_service_ = "deep_infra"
     _env_key_name_ = "DEEP_INFRA_API_KEY"
-    _base_url_ = "https://api.deepinfra.com/v1/openai" 
+    _base_url_ = "https://api.deepinfra.com/v1/openai"
     _models_list_cache: List[str] = []
-

edsl/inference_services/GoogleService.py

@@ -10,10 +10,16 @@ from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
 
 class GoogleService(InferenceServiceABC):
     _inference_service_ = "google"
+    key_sequence = ["candidates", 0, "content", "parts", 0, "text"]
+    usage_sequence = ["usageMetadata"]
+    input_token_name = "promptTokenCount"
+    output_token_name = "candidatesTokenCount"
+
+    model_exclude_list = []
 
     @classmethod
     def available(cls):
-        return ["gemini-pro"]
+        return ["gemini-pro", "gemini-1.5-pro", "gemini-1.5-flash", "gemini-1.0-pro"]
 
     @classmethod
     def create_model(
@@ -24,7 +30,15 @@ class GoogleService(InferenceServiceABC):
 
         class LLM(LanguageModel):
             _model_ = model_name
+            key_sequence = cls.key_sequence
+            usage_sequence = cls.usage_sequence
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
             _inference_service_ = cls._inference_service_
+
+            _tpm = cls.get_tpm(cls)
+            _rpm = cls.get_rpm(cls)
+
             _parameters_ = {
                 "temperature": 0.5,
                 "topP": 1,
@@ -50,7 +64,7 @@ class GoogleService(InferenceServiceABC):
                         "stopSequences": self.stopSequences,
                     },
                 }
-
+                # print(combined_prompt)
                 async with aiohttp.ClientSession() as session:
                     async with session.post(
                         url, headers=headers, data=json.dumps(data)
@@ -58,16 +72,6 @@ class GoogleService(InferenceServiceABC):
                         raw_response_text = await response.text()
                         return json.loads(raw_response_text)
 
-            def parse_response(self, raw_response: dict[str, Any]) -> str:
-                data = raw_response
-                try:
-                    return data["candidates"][0]["content"]["parts"][0]["text"]
-                except KeyError as e:
-                    print(
-                        f"The data return was {data}, which was missing the key 'candidates'"
-                    )
-                    raise e
-
         LLM.__name__ = model_name
 
         return LLM

edsl/inference_services/GroqService.py

@@ -10,10 +10,11 @@ class GroqService(OpenAIService):
     _inference_service_ = "groq"
     _env_key_name_ = "GROQ_API_KEY"
 
-    _sync_client_ =  groq.Groq
-    _async_client_ = groq.AsyncGroq 
+    _sync_client_ = groq.Groq
+    _async_client_ = groq.AsyncGroq
 
-    #_base_url_ = "https://api.deepinfra.com/v1/openai" 
+    model_exclude_list = ["whisper-large-v3", "distil-whisper-large-v3-en"]
+
+    # _base_url_ = "https://api.deepinfra.com/v1/openai"
     _base_url_ = None
     _models_list_cache: List[str] = []
-

edsl/inference_services/InferenceServiceABC.py

@@ -1,22 +1,77 @@
 from abc import abstractmethod, ABC
-from typing import Any
+import os
 import re
+from edsl.config import CONFIG
 
 
 class InferenceServiceABC(ABC):
-    """Abstract class for inference services."""
+    """
+    Abstract class for inference services.
+    Anthropic: https://docs.anthropic.com/en/api/rate-limits
+    """
+
+    default_levels = {
+        "google": {"tpm": 2_000_000, "rpm": 15},
+        "openai": {"tpm": 2_000_000, "rpm": 10_000},
+        "anthropic": {"tpm": 2_000_000, "rpm": 500},
+    }
+
+    def __init_subclass__(cls):
+        """
+        Check that the subclass has the required attributes.
+        - `key_sequence` attribute determines...
+        - `model_exclude_list` attribute determines...
+        """
+        if not hasattr(cls, "key_sequence"):
+            raise NotImplementedError(
+                f"Class {cls.__name__} must have a 'key_sequence' attribute."
+            )
+        if not hasattr(cls, "model_exclude_list"):
+            raise NotImplementedError(
+                f"Class {cls.__name__} must have a 'model_exclude_list' attribute."
+            )
+
+    @classmethod
+    def _get_limt(cls, limit_type: str) -> int:
+        key = f"EDSL_SERVICE_{limit_type.upper()}_{cls._inference_service_.upper()}"
+        if key in os.environ:
+            return int(os.getenv(key))
+
+        if cls._inference_service_ in cls.default_levels:
+            return int(cls.default_levels[cls._inference_service_][limit_type])
+
+        return int(CONFIG.get(f"EDSL_SERVICE_{limit_type.upper()}_BASELINE"))
+
+    def get_tpm(cls) -> int:
+        """
+        Returns the TPM for the service. If the service is not defined in the environment variables, it will return the baseline TPM.
+        """
+        return cls._get_limt(limit_type="tpm")
+
+    def get_rpm(cls):
+        """
+        Returns the RPM for the service. If the service is not defined in the environment variables, it will return the baseline RPM.
+        """
+        return cls._get_limt(limit_type="rpm")
 
     @abstractmethod
     def available() -> list[str]:
+        """
+        Returns a list of available models for the service.
+        """
         pass
 
     @abstractmethod
     def create_model():
+        """
+        Returns a LanguageModel object.
+        """
         pass
 
     @staticmethod
     def to_class_name(s):
-        """Convert a string to a valid class name.
+        """
+        Converts a string to a valid class name.
 
         >>> InferenceServiceABC.to_class_name("hello world")
         'HelloWorld'

edsl/inference_services/InferenceServicesCollection.py

@@ -15,18 +15,19 @@ class InferenceServicesCollection:
         cls.added_models[service_name].append(model_name)
 
     @staticmethod
-    def _get_service_available(service) -> list[str]:
+    def _get_service_available(service, warn: bool = False) -> list[str]:
         from_api = True
         try:
             service_models = service.available()
         except Exception as e:
-            warnings.warn(
-                f"""Error getting models for {service._inference_service_}. 
-                Check that you have properly stored your Expected Parrot API key and activated remote inference, or stored your own API keys for the language models that you want to use.
-                See https://docs.expectedparrot.com/en/latest/api_keys.html for instructions on storing API keys.
-                Relying on cache.""",
-                UserWarning,
-            )
+            if warn:
+                warnings.warn(
+                    f"""Error getting models for {service._inference_service_}. 
+                    Check that you have properly stored your Expected Parrot API key and activated remote inference, or stored your own API keys for the language models that you want to use.
+                    See https://docs.expectedparrot.com/en/latest/api_keys.html for instructions on storing API keys.
+                    Relying on cache.""",
+                    UserWarning,
+                )
             from edsl.inference_services.models_available_cache import models_available
 
             service_models = models_available.get(service._inference_service_, [])
@@ -60,4 +61,8 @@ class InferenceServicesCollection:
                 if service_name is None or service_name == service._inference_service_:
                     return service.create_model(model_name)
 
+        # if model_name == "test":
+        #     from edsl.language_models import LanguageModel
+        #     return LanguageModel(test = True)
+
         raise Exception(f"Model {model_name} not found in any of the services")