edsl 0.1.33.dev1__py3-none-any.whl → 0.1.33.dev2__py3-none-any.whl

edsl/coop/coop.py

@@ -53,6 +53,7 @@ class Coop:
         method: str,
         payload: Optional[dict[str, Any]] = None,
         params: Optional[dict[str, Any]] = None,
+        timeout: Optional[float] = 5,
     ) -> requests.Response:
         """
         Send a request to the server and return the response.
@@ -62,11 +63,16 @@ class Coop:
             method = method.upper()
             if method in ["GET", "DELETE"]:
                 response = requests.request(
-                    method, url, params=params, headers=self.headers
+                    method, url, params=params, headers=self.headers, timeout=timeout
                 )
             elif method in ["POST", "PATCH"]:
                 response = requests.request(
-                    method, url, params=params, json=payload, headers=self.headers
+                    method,
+                    url,
+                    params=params,
+                    json=payload,
+                    headers=self.headers,
+                    timeout=timeout,
                 )
             else:
                 raise Exception(f"Invalid {method=}.")
@@ -110,10 +116,18 @@ class Coop:
     def edsl_settings(self) -> dict:
         """
         Retrieve and return the EDSL settings stored on Coop.
+        If no response is received within 5 seconds, return an empty dict.
         """
-        response = self._send_server_request(uri="api/v0/edsl-settings", method="GET")
-        self._resolve_server_response(response)
-        return response.json()
+        from requests.exceptions import Timeout
+
+        try:
+            response = self._send_server_request(
+                uri="api/v0/edsl-settings", method="GET", timeout=5
+            )
+            self._resolve_server_response(response)
+            return response.json()
+        except Timeout:
+            return {}
 
     ################
     # Objects
@@ -625,6 +639,26 @@ class Coop:
 
         return response_json
 
+    def fetch_prices(self) -> dict:
+        from edsl.coop.PriceFetcher import PriceFetcher
+
+        from edsl.config import CONFIG
+
+        if bool(CONFIG.get("EDSL_FETCH_TOKEN_PRICES")):
+            price_fetcher = PriceFetcher()
+            return price_fetcher.fetch_prices()
+        else:
+            return {}
+
+
+if __name__ == "__main__":
+    sheet_data = fetch_sheet_data()
+    if sheet_data:
+        print(f"Successfully fetched {len(sheet_data)} rows of data.")
+        print("First row:", sheet_data[0])
+    else:
+        print("Failed to fetch sheet data.")
+
 
 def main():
     """

edsl/data/Cache.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 import json
 import os
 import warnings
+import copy
 from typing import Optional, Union
 from edsl.Base import Base
 from edsl.data.CacheEntry import CacheEntry
@@ -88,11 +89,24 @@ class Cache(Base):
         # raise NotImplementedError("This method is not implemented yet.")
 
     def keys(self):
+        """
+        >>> from edsl import Cache
+        >>> Cache.example().keys()
+        ['5513286eb6967abc0511211f0402587d']
+        """
         return list(self.data.keys())
 
     def values(self):
+        """
+        >>> from edsl import Cache
+        >>> Cache.example().values()
+        [CacheEntry(...)]
+        """
         return list(self.data.values())
 
+    def items(self):
+        return zip(self.keys(), self.values())
+
     def new_entries_cache(self) -> Cache:
         """Return a new Cache object with the new entries."""
         return Cache(data={**self.new_entries, **self.fetched_data})
@@ -160,7 +174,7 @@ class Cache(Base):
         parameters: str,
         system_prompt: str,
         user_prompt: str,
-        response: str,
+        response: dict,
         iteration: int,
     ) -> str:
         """
@@ -174,6 +188,15 @@ class Cache(Base):
         * The key-value pair is added to `self.new_entries`
         * If `immediate_write` is True , the key-value pair is added to `self.data`
         * If `immediate_write` is False, the key-value pair is added to `self.new_entries_to_write_later`
+
+        >>> from edsl import Cache, Model, Question
+        >>> m = Model("test")
+        >>> c = Cache()
+        >>> len(c)
+        0
+        >>> results = Question.example("free_text").by(m).run(cache = c)
+        >>> len(c)
+        1
         """
 
         entry = CacheEntry(
@@ -326,6 +349,17 @@ class Cache(Base):
             for key, value in self.data.items():
                 f.write(json.dumps({key: value.to_dict()}) + "\n")
 
+    def to_scenario_list(self):
+        from edsl import ScenarioList, Scenario
+
+        scenarios = []
+        for key, value in self.data.items():
+            new_d = value.to_dict()
+            new_d["cache_key"] = key
+            s = Scenario(new_d)
+            scenarios.append(s)
+        return ScenarioList(scenarios)
+
     ####################
     # REMOTE
     ####################

edsl/data_transfer_models.py

@@ -1,38 +1,120 @@
-"""This module contains the data transfer models for the application."""
-
-from collections import UserDict
-
-
-class AgentResponseDict(UserDict):
-    """A dictionary to store the response of the agent to a question."""
-
-    def __init__(
-        self,
-        *,
-        question_name,
-        answer,
-        prompts,
-        usage=None,
-        comment=None,
-        cached_response=None,
-        raw_model_response=None,
-        simple_model_raw_response=None,
-        cache_used=None,
-        cache_key=None,
-    ):
-        """Initialize the AgentResponseDict object."""
-        usage = usage or {"prompt_tokens": 0, "completion_tokens": 0}
-        super().__init__(
-            {
-                "answer": answer,
-                "comment": comment,
-                "question_name": question_name,
-                "prompts": prompts,
-                "usage": usage,
-                "cached_response": cached_response,
-                "raw_model_response": raw_model_response,
-                "simple_model_raw_response": simple_model_raw_response,
-                "cache_used": cache_used,
-                "cache_key": cache_key,
-            }
-        )
+from typing import NamedTuple, Dict, List, Optional, Any
+
+
+class ModelInputs(NamedTuple):
+    "This is what was send by the agent to the model"
+    user_prompt: str
+    system_prompt: str
+    encoded_image: Optional[str] = None
+
+
+class EDSLOutput(NamedTuple):
+    "This is the edsl dictionary that is returned by the model"
+    answer: Any
+    generated_tokens: str
+    comment: Optional[str] = None
+
+
+class ModelResponse(NamedTuple):
+    "This is the metadata that is returned by the model and includes info about the cache"
+    response: dict
+    cache_used: bool
+    cache_key: str
+    cached_response: Optional[Dict[str, Any]] = None
+    cost: Optional[float] = None
+
+
+class AgentResponseDict(NamedTuple):
+    edsl_dict: EDSLOutput
+    model_inputs: ModelInputs
+    model_outputs: ModelResponse
+
+
+class EDSLResultObjectInput(NamedTuple):
+    generated_tokens: str
+    question_name: str
+    prompts: dict
+    cached_response: str
+    raw_model_response: str
+    cache_used: bool
+    cache_key: str
+    answer: Any
+    comment: str
+    validated: bool = False
+    exception_occurred: Exception = None
+    cost: Optional[float] = None
+
+
+# from collections import UserDict
+
+
+# class AgentResponseDict(UserDict):
+#     """A dictionary to store the response of the agent to a question."""
+
+#     def __init__(
+#         self,
+#         *,
+#         question_name,
+#         answer,
+#         prompts,
+#         generated_tokens: str,
+#         usage=None,
+#         comment=None,
+#         cached_response=None,
+#         raw_model_response=None,
+#         simple_model_raw_response=None,
+#         cache_used=None,
+#         cache_key=None,
+#     ):
+#         """Initialize the AgentResponseDict object."""
+#         usage = usage or {"prompt_tokens": 0, "completion_tokens": 0}
+#         if generated_tokens is None:
+#             raise ValueError("generated_tokens must be provided")
+#         self.data = {
+#             "answer": answer,
+#             "comment": comment,
+#             "question_name": question_name,
+#             "prompts": prompts,
+#             "usage": usage,
+#             "cached_response": cached_response,
+#             "raw_model_response": raw_model_response,
+#             "simple_model_raw_response": simple_model_raw_response,
+#             "cache_used": cache_used,
+#             "cache_key": cache_key,
+#             "generated_tokens": generated_tokens,
+#         }
+
+#     @property
+#     def data(self):
+#         return self._data
+
+#     @data.setter
+#     def data(self, value):
+#         self._data = value
+
+#     def __getitem__(self, key):
+#         return self.data.get(key, None)
+
+#     def __setitem__(self, key, value):
+#         self.data[key] = value
+
+#     def __delitem__(self, key):
+#         del self.data[key]
+
+#     def __iter__(self):
+#         return iter(self.data)
+
+#     def __len__(self):
+#         return len(self.data)
+
+#     def keys(self):
+#         return self.data.keys()
+
+#     def values(self):
+#         return self.data.values()
+
+#     def items(self):
+#         return self.data.items()
+
+#     def is_this_same_model(self):
+#         return True

edsl/enums.py

@@ -62,6 +62,7 @@ class InferenceServiceType(EnumWithChecks):
     GROQ = "groq"
     AZURE = "azure"
     OLLAMA = "ollama"
+    MISTRAL = "mistral"
 
 
 service_to_api_keyname = {
@@ -74,6 +75,7 @@ service_to_api_keyname = {
     InferenceServiceType.ANTHROPIC.value: "ANTHROPIC_API_KEY",
     InferenceServiceType.GROQ.value: "GROQ_API_KEY",
     InferenceServiceType.BEDROCK.value: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"],
+    InferenceServiceType.MISTRAL.value: "MISTRAL_API_KEY",
 }
 
 

edsl/exceptions/language_models.py

@@ -1,8 +1,32 @@
 from textwrap import dedent
+from typing import Optional
 
 
 class LanguageModelExceptions(Exception):
-    pass
+    explanation = (
+        "This is the base class for all exceptions in the LanguageModel class."
+    )
+
+    def __init__(self, message):
+        super().__init__(message)
+        self.message = message
+
+
+class LanguageModelNoResponseError(LanguageModelExceptions):
+    explanation = (
+        """This happens when the LLM API cannot be reached and/or does not respond."""
+    )
+
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class LanguageModelBadResponseError(LanguageModelExceptions):
+    explanation = """This happens when the LLM API can be reached and responds, does not return a usable answer."""
+
+    def __init__(self, message, response_json: Optional[dict] = None):
+        super().__init__(message)
+        self.response_json = response_json
 
 
 class LanguageModelNotFound(LanguageModelExceptions):

edsl/exceptions/questions.py

@@ -1,16 +1,73 @@
+from typing import Any, SupportsIndex
+from jinja2 import Template
+import json
+
+
 class QuestionErrors(Exception):
-    pass
+    """
+    Base exception class for question-related errors.
+    """
 
+    def __init__(self, message="An error occurred with the question"):
+        self.message = message
+        super().__init__(self.message)
 
-class QuestionCreationValidationError(QuestionErrors):
-    pass
 
+class QuestionAnswerValidationError(QuestionErrors):
+    documentation = "https://docs.expectedparrot.com/en/latest/exceptions.html"
+
+    explanation = """This when the answer coming from the Language Model does not conform to the expectation for that question type.
+    For example, if the question is a multiple choice question, the answer should be drawn from the list of options provided.
+    """
+
+    def __init__(self, message="Invalid answer.", data=None, model=None):
+        self.message = message
+        self.data = data
+        self.model = model
+        super().__init__(self.message)
+
+    def __str__(self):
+        return f"""{repr(self)} 
+        Data being validated: {self.data} 
+        Pydnantic Model: {self.model}.
+        Reported error: {self.message}."""
+
+    def to_html_dict(self):
+        return {
+            "error_type": ("Name of the exception", "p", "/p", self.__class__.__name__),
+            "explaination": ("Explanation", "p", "/p", self.explanation),
+            "edsl answer": (
+                "What model returned",
+                "pre",
+                "/pre",
+                json.dumps(self.data, indent=2),
+            ),
+            "validating_model": (
+                "Pydantic model for answers",
+                "pre",
+                "/pre",
+                json.dumps(self.model.model_json_schema(), indent=2),
+            ),
+            "error_message": (
+                "Error message Pydantic returned",
+                "p",
+                "/p",
+                self.message,
+            ),
+            "documentation_url": (
+                "URL to EDSL docs",
+                f"a href='{self.documentation}'",
+                "/a",
+                self.documentation,
+            ),
+        }
 
-class QuestionResponseValidationError(QuestionErrors):
+
+class QuestionCreationValidationError(QuestionErrors):
     pass
 
 
-class QuestionAnswerValidationError(QuestionErrors):
+class QuestionResponseValidationError(QuestionErrors):
     pass
 
 

edsl/exceptions/results.py

@@ -2,6 +2,10 @@ class ResultsErrors(Exception):
     pass
 
 
+class ResultsDeserializationError(ResultsErrors):
+    pass
+
+
 class ResultsBadMutationstringError(ResultsErrors):
     pass
 

edsl/inference_services/AnthropicService.py

@@ -11,6 +11,11 @@ class AnthropicService(InferenceServiceABC):
 
     _inference_service_ = "anthropic"
     _env_key_name_ = "ANTHROPIC_API_KEY"
+    key_sequence = ["content", 0, "text"]  # ["content"][0]["text"]
+    usage_sequence = ["usage"]
+    input_token_name = "input_tokens"
+    output_token_name = "output_tokens"
+    model_exclude_list = []
 
     @classmethod
     def available(cls):
@@ -34,6 +39,11 @@ class AnthropicService(InferenceServiceABC):
             Child class of LanguageModel for interacting with OpenAI models
             """
 
+            key_sequence = cls.key_sequence
+            usage_sequence = cls.usage_sequence
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
+
             _inference_service_ = cls._inference_service_
             _model_ = model_name
             _parameters_ = {
@@ -46,6 +56,9 @@ class AnthropicService(InferenceServiceABC):
                 "top_logprobs": 3,
             }
 
+            _tpm = cls.get_tpm(cls)
+            _rpm = cls.get_rpm(cls)
+
             async def async_execute_model_call(
                 self, user_prompt: str, system_prompt: str = ""
             ) -> dict[str, Any]:
@@ -66,17 +79,6 @@ class AnthropicService(InferenceServiceABC):
                 )
                 return response.model_dump()
 
-            @staticmethod
-            def parse_response(raw_response: dict[str, Any]) -> str:
-                """Parses the API response and returns the response text."""
-                response = raw_response["content"][0]["text"]
-                pattern = r"^```json(?:\\n|\n)(.+?)(?:\\n|\n)```$"
-                match = re.match(pattern, response, re.DOTALL)
-                if match:
-                    return match.group(1)
-                else:
-                    return response
-
         LLM.__name__ = model_class_name
 
         return LLM

edsl/inference_services/AwsBedrock.py

@@ -16,6 +16,18 @@ class AwsBedrockService(InferenceServiceABC):
     _env_key_name_ = (
         "AWS_ACCESS_KEY_ID"  # or any other environment key for AWS credentials
     )
+    key_sequence = ["output", "message", "content", 0, "text"]
+    input_token_name = "inputTokens"
+    output_token_name = "outputTokens"
+    usage_sequence = ["usage"]
+    model_exclude_list = [
+        "ai21.j2-grande-instruct",
+        "ai21.j2-jumbo-instruct",
+        "ai21.j2-mid",
+        "ai21.j2-mid-v1",
+        "ai21.j2-ultra",
+        "ai21.j2-ultra-v1",
+    ]
 
     @classmethod
     def available(cls):
@@ -28,7 +40,7 @@ class AwsBedrockService(InferenceServiceABC):
         else:
             all_models_ids = cls._models_list_cache
 
-        return all_models_ids
+        return [m for m in all_models_ids if m not in cls.model_exclude_list]
 
     @classmethod
     def create_model(
@@ -42,6 +54,8 @@ class AwsBedrockService(InferenceServiceABC):
             Child class of LanguageModel for interacting with AWS Bedrock models.
             """
 
+            key_sequence = cls.key_sequence
+            usage_sequence = cls.usage_sequence
             _inference_service_ = cls._inference_service_
             _model_ = model_name
             _parameters_ = {
@@ -49,6 +63,10 @@ class AwsBedrockService(InferenceServiceABC):
                 "max_tokens": 512,
                 "top_p": 0.9,
             }
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
+            _rpm = cls.get_rpm(cls)
+            _tpm = cls.get_tpm(cls)
 
             async def async_execute_model_call(
                 self, user_prompt: str, system_prompt: str = ""
@@ -89,22 +107,6 @@ class AwsBedrockService(InferenceServiceABC):
                     print(e)
                     return {"error": str(e)}
 
-            @staticmethod
-            def parse_response(raw_response: dict[str, Any]) -> str:
-                """Parses the API response and returns the response text."""
-                if "output" in raw_response and "message" in raw_response["output"]:
-                    response = raw_response["output"]["message"]["content"][0]["text"]
-                    pattern = r"^```json(?:\\n|\n)(.+?)(?:\\n|\n)```$"
-                    match = re.match(pattern, response, re.DOTALL)
-                    if match:
-                        return match.group(1)
-                    else:
-                        out = fix_partial_correct_response(response)
-                        if "error" not in out:
-                            response = out["extracted_json"]
-                        return response
-                return "Error parsing response"
-
         LLM.__name__ = model_class_name
 
         return LLM

edsl/inference_services/AzureAI.py

@@ -25,11 +25,22 @@ def json_handle_none(value: Any) -> Any:
 class AzureAIService(InferenceServiceABC):
     """Azure AI service class."""
 
+    # key_sequence = ["content", 0, "text"]  # ["content"][0]["text"]
+    key_sequence = ["choices", 0, "message", "content"]
+    usage_sequence = ["usage"]
+    input_token_name = "prompt_tokens"
+    output_token_name = "completion_tokens"
+
     _inference_service_ = "azure"
     _env_key_name_ = (
         "AZURE_ENDPOINT_URL_AND_KEY"  # Environment variable for Azure API key
     )
     _model_id_to_endpoint_and_key = {}
+    model_exclude_list = [
+        "Cohere-command-r-plus-xncmg",
+        "Mistral-Nemo-klfsi",
+        "Mistral-large-2407-ojfld",
+    ]
 
     @classmethod
     def available(cls):
@@ -82,7 +93,7 @@ class AzureAIService(InferenceServiceABC):
 
             except Exception as e:
                 raise e
-        return out
+        return [m for m in out if m not in cls.model_exclude_list]
 
     @classmethod
     def create_model(
@@ -96,6 +107,10 @@ class AzureAIService(InferenceServiceABC):
             Child class of LanguageModel for interacting with Azure OpenAI models.
             """
 
+            key_sequence = cls.key_sequence
+            usage_sequence = cls.usage_sequence
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
             _inference_service_ = cls._inference_service_
             _model_ = model_name
             _parameters_ = {
@@ -103,6 +118,8 @@ class AzureAIService(InferenceServiceABC):
                 "max_tokens": 512,
                 "top_p": 0.9,
             }
+            _rpm = cls.get_rpm(cls)
+            _tpm = cls.get_tpm(cls)
 
             async def async_execute_model_call(
                 self, user_prompt: str, system_prompt: str = ""
@@ -172,25 +189,25 @@ class AzureAIService(InferenceServiceABC):
                     )
                     return response.model_dump()
 
-            @staticmethod
-            def parse_response(raw_response: dict[str, Any]) -> str:
-                """Parses the API response and returns the response text."""
-                if (
-                    raw_response
-                    and "choices" in raw_response
-                    and raw_response["choices"]
-                ):
-                    response = raw_response["choices"][0]["message"]["content"]
-                    pattern = r"^```json(?:\\n|\n)(.+?)(?:\\n|\n)```$"
-                    match = re.match(pattern, response, re.DOTALL)
-                    if match:
-                        return match.group(1)
-                    else:
-                        out = fix_partial_correct_response(response)
-                        if "error" not in out:
-                            response = out["extracted_json"]
-                        return response
-                return "Error parsing response"
+            # @staticmethod
+            # def parse_response(raw_response: dict[str, Any]) -> str:
+            #     """Parses the API response and returns the response text."""
+            #     if (
+            #         raw_response
+            #         and "choices" in raw_response
+            #         and raw_response["choices"]
+            #     ):
+            #         response = raw_response["choices"][0]["message"]["content"]
+            #         pattern = r"^```json(?:\\n|\n)(.+?)(?:\\n|\n)```$"
+            #         match = re.match(pattern, response, re.DOTALL)
+            #         if match:
+            #             return match.group(1)
+            #         else:
+            #             out = fix_partial_correct_response(response)
+            #             if "error" not in out:
+            #                 response = out["extracted_json"]
+            #             return response
+            #     return "Error parsing response"
 
         LLM.__name__ = model_class_name