edsl 0.1.32__py3-none-any.whl → 0.1.33__py3-none-any.whl

edsl/config.py

@@ -1,83 +1,66 @@
 """This module provides a Config class that loads environment variables from a .env file and sets them as class attributes."""
 
 import os
+from dotenv import load_dotenv, find_dotenv
 from edsl.exceptions import (
     InvalidEnvironmentVariableError,
     MissingEnvironmentVariableError,
 )
-from dotenv import load_dotenv, find_dotenv
 
 # valid values for EDSL_RUN_MODE
-EDSL_RUN_MODES = ["development", "development-testrun", "production"]
+EDSL_RUN_MODES = [
+    "development",
+    "development-testrun",
+    "production",
+]
 
 # `default` is used to impute values only in "production" mode
 # `info` gives a brief description of the env var
 CONFIG_MAP = {
     "EDSL_RUN_MODE": {
         "default": "production",
-        "info": "This env var determines the run mode of the application.",
-    },
-    "EDSL_DATABASE_PATH": {
-        "default": f"sqlite:///{os.path.join(os.getcwd(), '.edsl_cache/data.db')}",
-        "info": "This env var determines the path to the cache file.",
-    },
-    "EDSL_LOGGING_PATH": {
-        "default": f"{os.path.join(os.getcwd(), 'interview.log')}",
-        "info": "This env var determines the path to the log file.",
+        "info": "This config var determines the run mode of the application.",
     },
     "EDSL_API_TIMEOUT": {
         "default": "60",
-        "info": "This env var determines the maximum number of seconds to wait for an API call to return.",
+        "info": "This config var determines the maximum number of seconds to wait for an API call to return.",
     },
     "EDSL_BACKOFF_START_SEC": {
         "default": "1",
-        "info": "This env var determines the number of seconds to wait before retrying a failed API call.",
+        "info": "This config var determines the number of seconds to wait before retrying a failed API call.",
     },
-    "EDSL_MAX_BACKOFF_SEC": {
+    "EDSL_BACKOFF_MAX_SEC": {
         "default": "60",
-        "info": "This env var determines the maximum number of seconds to wait before retrying a failed API call.",
+        "info": "This config var determines the maximum number of seconds to wait before retrying a failed API call.",
+    },
+    "EDSL_DATABASE_PATH": {
+        "default": f"sqlite:///{os.path.join(os.getcwd(), '.edsl_cache/data.db')}",
+        "info": "This config var determines the path to the cache file.",
+    },
+    "EDSL_DEFAULT_MODEL": {
+        "default": "gpt-4o",
+        "info": "This config var holds the default model that will be used if a model is not explicitly passed.",
+    },
+    "EDSL_FETCH_TOKEN_PRICES": {
+        "default": "True",
+        "info": "This config var determines whether to fetch prices for tokens used in remote inference",
     },
     "EDSL_MAX_ATTEMPTS": {
         "default": "5",
-        "info": "This env var determines the maximum number of times to retry a failed API call.",
+        "info": "This config var determines the maximum number of times to retry a failed API call.",
+    },
+    "EDSL_SERVICE_RPM_BASELINE": {
+        "default": "100",
+        "info": "This config var holds the maximum number of requests per minute. Model-specific values provided in env vars such as EDSL_SERVICE_RPM_OPENAI will override this. value for the corresponding model",
+    },
+    "EDSL_SERVICE_TPM_BASELINE": {
+        "default": "2000000",
+        "info": "This config var holds the maximum number of tokens per minute for all models. Model-specific values provided in env vars such as EDSL_SERVICE_TPM_OPENAI will override this value for the corresponding model.",
     },
     "EXPECTED_PARROT_URL": {
         "default": "https://www.expectedparrot.com",
-        "info": "This env var holds the URL of the Expected Parrot API.",
+        "info": "This config var holds the URL of the Expected Parrot API.",
     },
-    # "EXPECTED_PARROT_API_KEY": {
-    #     "default": None,
-    #     "info": "This env var holds your Expected Parrot API key (https://www.expectedparrot.com/).",
-    # },
-    # "OPENAI_API_KEY": {
-    #     "default": None,
-    #     "info": "This env var holds your OpenAI API key (https://platform.openai.com/api-keys).",
-    # },
-    # "DEEP_INFRA_API_KEY": {
-    #     "default": None,
-    #     "info": "This env var holds your DeepInfra API key (https://deepinfra.com/).",
-    # },
-    # "GOOGLE_API_KEY": {
-    #     "default": None,
-    #     "info": "This env var holds your Google API key (https://console.cloud.google.com/apis/credentials).",
-    # },
-    # "ANTHROPIC_API_KEY": {
-    #     "default": None,
-    #     "info": "This env var holds your Anthropic API key (https://www.anthropic.com/).",
-    # },
-    # "GROQ_API_KEY": {
-    #     "default": None,
-    #     "info": "This env var holds your GROQ API key (https://console.groq.com/login).",
-    # },
-    # "AWS_ACCESS_KEY_ID" :
-    #     "default": None,
-    #     "info": "This env var holds your AWS access key ID.",
-    # "AWS_SECRET_ACCESS_KEY:
-    #     "default": None,
-    #     "info": "This env var holds your AWS secret access key.",
-    # "AZURE_ENDPOINT_URL_AND_KEY":
-    #     "default": None,
-    #     "info": "This env var holds your Azure endpoint URL and key (URL:key). You can have several comma-separated URL-key pairs (URL1:key1,URL2:key2).",
 }
 
 
@@ -92,7 +75,7 @@ class Config:
 
     def _set_run_mode(self) -> None:
         """
-        Checks the validity and sets EDSL_RUN_MODE.
+        Sets EDSL_RUN_MODE as a class attribute.
         """
         run_mode = os.getenv("EDSL_RUN_MODE")
         default = CONFIG_MAP.get("EDSL_RUN_MODE").get("default")
@@ -107,27 +90,35 @@ class Config:
     def _load_dotenv(self) -> None:
         """
         Loads the .env
-        - Overrides existing env vars unless EDSL_RUN_MODE=="development-testrun"
+        - The .env will override existing env vars **unless** EDSL_RUN_MODE=="development-testrun"
         """
 
-        override = True
         if self.EDSL_RUN_MODE == "development-testrun":
             override = False
+        else:
+            override = True
         _ = load_dotenv(dotenv_path=find_dotenv(usecwd=True), override=override)
 
+    def __contains__(self, env_var: str) -> bool:
+        """
+        Checks if an env var is set as a class attribute.
+        """
+        return env_var in self.__dict__
+
     def _set_env_vars(self) -> None:
         """
-        Sets env vars as Config class attributes.
+        Sets env vars as class attributes.
+        - EDSL_RUN_MODE is not set my this method, but by _set_run_mode
         - If an env var is not set and has a default value in the CONFIG_MAP, sets it to the default value.
         """
         # for each env var in the CONFIG_MAP
         for env_var, config in CONFIG_MAP.items():
-            # we've set it already in _set_run_mode
+            # EDSL_RUN_MODE is already set by _set_run_mode
             if env_var == "EDSL_RUN_MODE":
                 continue
             value = os.getenv(env_var)
             default_value = config.get("default")
-            # if the env var is set, set it as a CONFIG attribute
+            # if an env var exists, set it as a class attribute
             if value:
                 setattr(self, env_var, value)
             # otherwise, if EDSL_RUN_MODE == "production" set it to its default value

edsl/coop/PriceFetcher.py

@@ -0,0 +1,58 @@
+import requests
+import csv
+from io import StringIO
+
+
+class PriceFetcher:
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(PriceFetcher, cls).__new__(cls)
+            cls._instance._cached_prices = None
+        return cls._instance
+
+    def fetch_prices(self):
+        if self._cached_prices is not None:
+            return self._cached_prices
+
+        import requests
+        import csv
+        from io import StringIO
+
+        sheet_id = "1SAO3Bhntefl0XQHJv27rMxpvu6uzKDWNXFHRa7jrUDs"
+
+        # Construct the URL to fetch the CSV
+        url = f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv"
+
+        try:
+            # Fetch the CSV data
+            response = requests.get(url)
+            response.raise_for_status()  # Raise an exception for bad responses
+
+            # Parse the CSV data
+            csv_data = StringIO(response.text)
+            reader = csv.reader(csv_data)
+
+            # Convert to list of dictionaries
+            headers = next(reader)
+            data = [dict(zip(headers, row)) for row in reader]
+
+            # self._cached_prices = data
+            # return data
+            price_lookup = {}
+            for entry in data:
+                service = entry.get("service", None)
+                model = entry.get("model", None)
+                if service and model:
+                    token_type = entry.get("token_type", None)
+                    if (service, model) in price_lookup:
+                        price_lookup[(service, model)].update({token_type: entry})
+                    else:
+                        price_lookup[(service, model)] = {token_type: entry}
+            self._cached_prices = price_lookup
+            return self._cached_prices
+
+        except requests.RequestException as e:
+            # print(f"An error occurred: {e}")
+            return {}

edsl/coop/coop.py

@@ -53,25 +53,39 @@ class Coop:
         method: str,
         payload: Optional[dict[str, Any]] = None,
         params: Optional[dict[str, Any]] = None,
+        timeout: Optional[float] = 5,
     ) -> requests.Response:
         """
         Send a request to the server and return the response.
         """
         url = f"{self.url}/{uri}"
+        method = method.upper()
+        if payload is None:
+            timeout = 20
+        elif (
+            method.upper() == "POST"
+            and "json_string" in payload
+            and payload.get("json_string") is not None
+        ):
+            timeout = max(20, (len(payload.get("json_string", "")) // (1024 * 1024)))
         try:
-            method = method.upper()
             if method in ["GET", "DELETE"]:
                 response = requests.request(
-                    method, url, params=params, headers=self.headers
+                    method, url, params=params, headers=self.headers, timeout=timeout
                 )
             elif method in ["POST", "PATCH"]:
                 response = requests.request(
-                    method, url, params=params, json=payload, headers=self.headers
+                    method,
+                    url,
+                    params=params,
+                    json=payload,
+                    headers=self.headers,
+                    timeout=timeout,
                 )
             else:
                 raise Exception(f"Invalid {method=}.")
         except requests.ConnectionError:
-            raise requests.ConnectionError("Could not connect to the server.")
+            raise requests.ConnectionError(f"Could not connect to the server at {url}.")
 
         return response
 
@@ -81,6 +95,7 @@ class Coop:
         """
         if response.status_code >= 400:
             message = response.json().get("detail")
+            # print(response.text)
             if "Authorization" in message:
                 print(message)
                 message = "Please provide an Expected Parrot API key."
@@ -110,10 +125,18 @@ class Coop:
     def edsl_settings(self) -> dict:
         """
         Retrieve and return the EDSL settings stored on Coop.
+        If no response is received within 5 seconds, return an empty dict.
         """
-        response = self._send_server_request(uri="api/v0/edsl-settings", method="GET")
-        self._resolve_server_response(response)
-        return response.json()
+        from requests.exceptions import Timeout
+
+        try:
+            response = self._send_server_request(
+                uri="api/v0/edsl-settings", method="GET", timeout=5
+            )
+            self._resolve_server_response(response)
+            return response.json()
+        except Timeout:
+            return {}
 
     ################
     # Objects
@@ -625,6 +648,26 @@ class Coop:
 
         return response_json
 
+    def fetch_prices(self) -> dict:
+        from edsl.coop.PriceFetcher import PriceFetcher
+
+        from edsl.config import CONFIG
+
+        if bool(CONFIG.get("EDSL_FETCH_TOKEN_PRICES")):
+            price_fetcher = PriceFetcher()
+            return price_fetcher.fetch_prices()
+        else:
+            return {}
+
+
+if __name__ == "__main__":
+    sheet_data = fetch_sheet_data()
+    if sheet_data:
+        print(f"Successfully fetched {len(sheet_data)} rows of data.")
+        print("First row:", sheet_data[0])
+    else:
+        print("Failed to fetch sheet data.")
+
 
 def main():
     """

edsl/data/Cache.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 import json
 import os
 import warnings
+import copy
 from typing import Optional, Union
 from edsl.Base import Base
 from edsl.data.CacheEntry import CacheEntry
@@ -88,11 +89,24 @@ class Cache(Base):
         # raise NotImplementedError("This method is not implemented yet.")
 
     def keys(self):
+        """
+        >>> from edsl import Cache
+        >>> Cache.example().keys()
+        ['5513286eb6967abc0511211f0402587d']
+        """
         return list(self.data.keys())
 
     def values(self):
+        """
+        >>> from edsl import Cache
+        >>> Cache.example().values()
+        [CacheEntry(...)]
+        """
         return list(self.data.values())
 
+    def items(self):
+        return zip(self.keys(), self.values())
+
     def new_entries_cache(self) -> Cache:
         """Return a new Cache object with the new entries."""
         return Cache(data={**self.new_entries, **self.fetched_data})
@@ -160,7 +174,7 @@ class Cache(Base):
         parameters: str,
         system_prompt: str,
         user_prompt: str,
-        response: str,
+        response: dict,
         iteration: int,
     ) -> str:
         """
@@ -174,6 +188,15 @@ class Cache(Base):
         * The key-value pair is added to `self.new_entries`
         * If `immediate_write` is True , the key-value pair is added to `self.data`
         * If `immediate_write` is False, the key-value pair is added to `self.new_entries_to_write_later`
+
+        >>> from edsl import Cache, Model, Question
+        >>> m = Model("test")
+        >>> c = Cache()
+        >>> len(c)
+        0
+        >>> results = Question.example("free_text").by(m).run(cache = c)
+        >>> len(c)
+        1
         """
 
         entry = CacheEntry(
@@ -326,6 +349,17 @@ class Cache(Base):
             for key, value in self.data.items():
                 f.write(json.dumps({key: value.to_dict()}) + "\n")
 
+    def to_scenario_list(self):
+        from edsl import ScenarioList, Scenario
+
+        scenarios = []
+        for key, value in self.data.items():
+            new_d = value.to_dict()
+            new_d["cache_key"] = key
+            s = Scenario(new_d)
+            scenarios.append(s)
+        return ScenarioList(scenarios)
+
     ####################
     # REMOTE
     ####################

edsl/data_transfer_models.py

@@ -1,38 +1,73 @@
-"""This module contains the data transfer models for the application."""
-
-from collections import UserDict
-
-
-class AgentResponseDict(UserDict):
-    """A dictionary to store the response of the agent to a question."""
-
-    def __init__(
-        self,
-        *,
-        question_name,
-        answer,
-        prompts,
-        usage=None,
-        comment=None,
-        cached_response=None,
-        raw_model_response=None,
-        simple_model_raw_response=None,
-        cache_used=None,
-        cache_key=None,
-    ):
-        """Initialize the AgentResponseDict object."""
-        usage = usage or {"prompt_tokens": 0, "completion_tokens": 0}
-        super().__init__(
-            {
-                "answer": answer,
-                "comment": comment,
-                "question_name": question_name,
-                "prompts": prompts,
-                "usage": usage,
-                "cached_response": cached_response,
-                "raw_model_response": raw_model_response,
-                "simple_model_raw_response": simple_model_raw_response,
-                "cache_used": cache_used,
-                "cache_key": cache_key,
-            }
-        )
+from typing import NamedTuple, Dict, List, Optional, Any
+from dataclasses import dataclass, fields
+import reprlib
+
+
+class ModelInputs(NamedTuple):
+    "This is what was send by the agent to the model"
+    user_prompt: str
+    system_prompt: str
+    encoded_image: Optional[str] = None
+
+
+class EDSLOutput(NamedTuple):
+    "This is the edsl dictionary that is returned by the model"
+    answer: Any
+    generated_tokens: str
+    comment: Optional[str] = None
+
+
+class ModelResponse(NamedTuple):
+    "This is the metadata that is returned by the model and includes info about the cache"
+    response: dict
+    cache_used: bool
+    cache_key: str
+    cached_response: Optional[Dict[str, Any]] = None
+    cost: Optional[float] = None
+
+
+class AgentResponseDict(NamedTuple):
+    edsl_dict: EDSLOutput
+    model_inputs: ModelInputs
+    model_outputs: ModelResponse
+
+
+class EDSLResultObjectInput(NamedTuple):
+    generated_tokens: str
+    question_name: str
+    prompts: dict
+    cached_response: str
+    raw_model_response: str
+    cache_used: bool
+    cache_key: str
+    answer: Any
+    comment: str
+    validated: bool = False
+    exception_occurred: Exception = None
+    cost: Optional[float] = None
+
+
+@dataclass
+class ImageInfo:
+    file_path: str
+    file_name: str
+    image_format: str
+    file_size: int
+    encoded_image: str
+
+    def __repr__(self):
+        reprlib_instance = reprlib.Repr()
+        reprlib_instance.maxstring = 30  # Limit the string length for the encoded image
+
+        # Get all fields except encoded_image
+        field_reprs = [
+            f"{f.name}={getattr(self, f.name)!r}"
+            for f in fields(self)
+            if f.name != "encoded_image"
+        ]
+
+        # Add the reprlib-restricted encoded_image field
+        field_reprs.append(f"encoded_image={reprlib_instance.repr(self.encoded_image)}")
+
+        # Join everything to create the repr
+        return f"{self.__class__.__name__}({', '.join(field_reprs)})"

edsl/enums.py

@@ -62,6 +62,8 @@ class InferenceServiceType(EnumWithChecks):
     GROQ = "groq"
     AZURE = "azure"
     OLLAMA = "ollama"
+    MISTRAL = "mistral"
+    TOGETHER = "together"
 
 
 service_to_api_keyname = {
@@ -74,6 +76,8 @@ service_to_api_keyname = {
     InferenceServiceType.ANTHROPIC.value: "ANTHROPIC_API_KEY",
     InferenceServiceType.GROQ.value: "GROQ_API_KEY",
     InferenceServiceType.BEDROCK.value: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"],
+    InferenceServiceType.MISTRAL.value: "MISTRAL_API_KEY",
+    InferenceServiceType.TOGETHER.value: "TOGETHER_API_KEY",
 }
 
 

edsl/exceptions/language_models.py

@@ -1,8 +1,32 @@
 from textwrap import dedent
+from typing import Optional
 
 
 class LanguageModelExceptions(Exception):
-    pass
+    explanation = (
+        "This is the base class for all exceptions in the LanguageModel class."
+    )
+
+    def __init__(self, message):
+        super().__init__(message)
+        self.message = message
+
+
+class LanguageModelNoResponseError(LanguageModelExceptions):
+    explanation = (
+        """This happens when the LLM API cannot be reached and/or does not respond."""
+    )
+
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class LanguageModelBadResponseError(LanguageModelExceptions):
+    explanation = """This happens when the LLM API can be reached and responds, does not return a usable answer."""
+
+    def __init__(self, message, response_json: Optional[dict] = None):
+        super().__init__(message)
+        self.response_json = response_json
 
 
 class LanguageModelNotFound(LanguageModelExceptions):

edsl/exceptions/questions.py

@@ -1,16 +1,73 @@
+from typing import Any, SupportsIndex
+from jinja2 import Template
+import json
+
+
 class QuestionErrors(Exception):
-    pass
+    """
+    Base exception class for question-related errors.
+    """
 
+    def __init__(self, message="An error occurred with the question"):
+        self.message = message
+        super().__init__(self.message)
 
-class QuestionCreationValidationError(QuestionErrors):
-    pass
 
+class QuestionAnswerValidationError(QuestionErrors):
+    documentation = "https://docs.expectedparrot.com/en/latest/exceptions.html"
+
+    explanation = """This when the answer coming from the Language Model does not conform to the expectation for that question type.
+    For example, if the question is a multiple choice question, the answer should be drawn from the list of options provided.
+    """
+
+    def __init__(self, message="Invalid answer.", data=None, model=None):
+        self.message = message
+        self.data = data
+        self.model = model
+        super().__init__(self.message)
+
+    def __str__(self):
+        return f"""{repr(self)} 
+        Data being validated: {self.data} 
+        Pydnantic Model: {self.model}.
+        Reported error: {self.message}."""
+
+    def to_html_dict(self):
+        return {
+            "error_type": ("Name of the exception", "p", "/p", self.__class__.__name__),
+            "explaination": ("Explanation", "p", "/p", self.explanation),
+            "edsl answer": (
+                "What model returned",
+                "pre",
+                "/pre",
+                json.dumps(self.data, indent=2),
+            ),
+            "validating_model": (
+                "Pydantic model for answers",
+                "pre",
+                "/pre",
+                json.dumps(self.model.model_json_schema(), indent=2),
+            ),
+            "error_message": (
+                "Error message Pydantic returned",
+                "p",
+                "/p",
+                self.message,
+            ),
+            "documentation_url": (
+                "URL to EDSL docs",
+                f"a href='{self.documentation}'",
+                "/a",
+                self.documentation,
+            ),
+        }
 
-class QuestionResponseValidationError(QuestionErrors):
+
+class QuestionCreationValidationError(QuestionErrors):
     pass
 
 
-class QuestionAnswerValidationError(QuestionErrors):
+class QuestionResponseValidationError(QuestionErrors):
     pass
 
 

edsl/exceptions/results.py

@@ -2,6 +2,10 @@ class ResultsErrors(Exception):
     pass
 
 
+class ResultsDeserializationError(ResultsErrors):
+    pass
+
+
 class ResultsBadMutationstringError(ResultsErrors):
     pass