edsl 0.1.35__tar.gz → 0.1.36__tar.gz

{edsl-0.1.35 → edsl-0.1.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edsl
-Version: 0.1.35
+Version: 0.1.36
 Summary: Create and analyze LLM-based surveys
 Home-page: https://www.expectedparrot.com/
 License: MIT

{edsl-0.1.35 → edsl-0.1.36}/edsl/Base.py

@@ -7,6 +7,8 @@ import json
 from typing import Any, Optional, Union
 from uuid import UUID
 
+# from edsl.utilities.MethodSuggesterMixin import MethodSuggesterMixin
+
 
 class RichPrintingMixin:
     """Mixin for rich printing and persistence of objects."""
@@ -274,6 +276,9 @@ class Base(
         """This method should be implemented by subclasses."""
         raise NotImplementedError("This method is not implemented yet.")
 
+    def to_json(self):
+        return json.dumps(self.to_dict())
+
     @abstractmethod
     def from_dict():
         """This method should be implemented by subclasses."""

{edsl-0.1.35 → edsl-0.1.36}/edsl/__init__.py

@@ -27,6 +27,7 @@ from edsl.questions import QuestionTopK
 
 from edsl.scenarios import Scenario
 from edsl.scenarios import ScenarioList
+from edsl.scenarios.FileStore import FileStore
 
 # from edsl.utilities.interface import print_dict_with_rich
 from edsl.surveys.Survey import Survey

edsl-0.1.36/edsl/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.36"

{edsl-0.1.35 → edsl-0.1.36}/edsl/agents/Agent.py

@@ -8,6 +8,9 @@ from typing import Callable, Optional, Union, Any
 from uuid import uuid4
 from edsl.Base import Base
 
+from edsl.prompts import Prompt
+from edsl.exceptions import QuestionScenarioRenderError
+
 from edsl.exceptions.agents import (
     AgentCombinationError,
     AgentDirectAnswerFunctionError,
@@ -44,7 +47,6 @@ class Agent(Base):
 
     def __init__(
         self,
-        # *,
         traits: Optional[dict] = None,
         name: Optional[str] = None,
         codebook: Optional[dict] = None,
@@ -109,9 +111,14 @@ class Agent(Base):
         self.name = name
         self._traits = traits or dict()
         self.codebook = codebook or dict()
-        self.instruction = instruction or self.default_instruction
+        if instruction is None:
+            self.instruction = self.default_instruction
+        else:
+            self.instruction = instruction
+        # self.instruction = instruction or self.default_instruction
         self.dynamic_traits_function = dynamic_traits_function
 
+        # Deal with dynamic traits function
         if self.dynamic_traits_function:
             self.dynamic_traits_function_name = self.dynamic_traits_function.__name__
             self.has_dynamic_traits_function = True
@@ -124,6 +131,7 @@ class Agent(Base):
                 dynamic_traits_function_name, dynamic_traits_function
             )
 
+        # Deal with direct answer function
         if answer_question_directly_source_code:
             self.answer_question_directly_function_name = (
                 answer_question_directly_function_name
@@ -140,10 +148,34 @@ class Agent(Base):
         self.current_question = None
 
         if traits_presentation_template is not None:
-            from edsl.prompts.library.agent_persona import AgentPersona
-
             self.traits_presentation_template = traits_presentation_template
-            self.agent_persona = AgentPersona(text=self.traits_presentation_template)
+        else:
+            self.traits_presentation_template = """Your traits: {{ traits }}"""
+
+    @property
+    def agent_persona(self) -> Prompt:
+        return Prompt(text=self.traits_presentation_template)
+
+    def prompt(self) -> str:
+        """Return the prompt for the agent.
+
+        Example usage:
+
+        >>> a = Agent(traits = {"age": 10, "hair": "brown", "height": 5.5})
+        >>> a.prompt()
+        Prompt(text=\"""Your traits: {'age': 10, 'hair': 'brown', 'height': 5.5}\""")
+        """
+        replacement_dict = (
+            self.traits | {"traits": self.traits} | {"codebook": self.codebook}
+        )
+        if undefined := self.agent_persona.undefined_template_variables(
+            replacement_dict
+        ):
+            raise QuestionScenarioRenderError(
+                f"Agent persona still has variables that were not rendered: {undefined}"
+            )
+        else:
+            return self.agent_persona.render(replacement_dict)
 
     def _check_dynamic_traits_function(self) -> None:
         """Check whether dynamic trait function is valid.
@@ -252,7 +284,6 @@ class Agent(Base):
             warnings.warn(
                 "Warning: overwriting existing answer_question_directly method"
             )
-            # print("Warning: overwriting existing answer_question_directly method")
 
         self.validate_response = validate_response
         self.translate_response = translate_response
@@ -575,9 +606,6 @@ class Agent(Base):
         if self.name == None:
             raw_data.pop("name")
 
-        import inspect
-
-        # print(raw_data)
         if hasattr(self, "dynamic_traits_function"):
             raw_data.pop(
                 "dynamic_traits_function", None

{edsl-0.1.35 → edsl-0.1.36}/edsl/agents/Invigilator.py

@@ -4,7 +4,8 @@ from typing import Dict, Any, Optional
 
 from edsl.prompts.Prompt import Prompt
 from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
-from edsl.prompts.registry import get_classes as prompt_lookup
+
+# from edsl.prompts.registry import get_classes as prompt_lookup
 from edsl.exceptions.questions import QuestionAnswerValidationError
 from edsl.agents.InvigilatorBase import InvigilatorBase
 from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput

{edsl-0.1.35 → edsl-0.1.36}/edsl/agents/InvigilatorBase.py

@@ -115,7 +115,11 @@ class InvigilatorBase(ABC):
         iteration = data["iteration"]
         additional_prompt_data = data["additional_prompt_data"]
         cache = Cache.from_dict(data["cache"])
-        sidecar_model = LanguageModel.from_dict(data["sidecar_model"])
+
+        if data["sidecar_model"] is None:
+            sidecar_model = None
+        else:
+            sidecar_model = LanguageModel.from_dict(data["sidecar_model"])
 
         return cls(
             agent=agent,

{edsl-0.1.35 → edsl-0.1.36}/edsl/agents/PromptConstructor.py

@@ -1,16 +1,11 @@
 from __future__ import annotations
 from typing import Dict, Any, Optional, Set
-from collections import UserList
-import pdb
 
 from jinja2 import Environment, meta
 
 from edsl.prompts.Prompt import Prompt
-from edsl.data_transfer_models import ImageInfo
-from edsl.prompts.registry import get_classes as prompt_lookup
-from edsl.exceptions import QuestionScenarioRenderError
 
-from edsl.agents.prompt_helpers import PromptComponent, PromptList, PromptPlan
+from edsl.agents.prompt_helpers import PromptPlan
 
 
 def get_jinja2_variables(template_str: str) -> Set[str]:
@@ -75,17 +70,8 @@ class PromptConstructor:
 
         if self.agent == Agent():  # if agent is empty, then return an empty prompt
             return Prompt(text="")
-        if not hasattr(self, "_agent_instructions_prompt"):
-            applicable_prompts = prompt_lookup(
-                component_type="agent_instructions",
-                model=self.model.model,
-            )
-            if len(applicable_prompts) == 0:
-                raise Exception("No applicable prompts found")
-            self._agent_instructions_prompt = applicable_prompts[0](
-                text=self.agent.instruction
-            )
-        return self._agent_instructions_prompt
+
+        return Prompt(text=self.agent.instruction)
 
     @property
     def agent_persona_prompt(self) -> Prompt:
@@ -93,51 +79,14 @@ class PromptConstructor:
         >>> from edsl.agents.InvigilatorBase import InvigilatorBase
         >>> i = InvigilatorBase.example()
         >>> i.prompt_constructor.agent_persona_prompt
-        Prompt(text=\"""You are an agent with the following persona:
-        {'age': 22, 'hair': 'brown', 'height': 5.5}\""")
-
+        Prompt(text=\"""Your traits: {'age': 22, 'hair': 'brown', 'height': 5.5}\""")
         """
         from edsl import Agent
 
-        if hasattr(self, "_agent_persona_prompt"):
-            return self._agent_persona_prompt
-
         if self.agent == Agent():  # if agent is empty, then return an empty prompt
             return Prompt(text="")
 
-        if not hasattr(self.agent, "agent_persona"):
-            applicable_prompts = prompt_lookup(
-                component_type="agent_persona",
-                model=self.model.model,
-            )
-            persona_prompt_template = applicable_prompts[0]()
-        else:
-            persona_prompt_template = self.agent.agent_persona
-
-        # TODO: This multiple passing of agent traits - not sure if it is necessary. Not harmful.
-        template_parameter_dictionary = (
-            self.agent.traits
-            | {"traits": self.agent.traits}
-            | {"codebook": self.agent.codebook}
-            | {"traits": self.agent.traits}
-        )
-
-        if undefined := persona_prompt_template.undefined_template_variables(
-            template_parameter_dictionary
-        ):
-            raise QuestionScenarioRenderError(
-                f"Agent persona still has variables that were not rendered: {undefined}"
-            )
-
-        persona_prompt = persona_prompt_template.render(template_parameter_dictionary)
-        if persona_prompt.has_variables:
-            raise QuestionScenarioRenderError(
-                "Agent persona still has variables that were not rendered."
-            )
-
-        self._agent_persona_prompt = persona_prompt
-
-        return self._agent_persona_prompt
+        return self.agent.prompt()
 
     def prior_answers_dict(self) -> dict:
         d = self.survey.question_names_to_questions()
@@ -198,14 +147,28 @@ class PromptConstructor:
 
                 # might be getting it from the prior answers
                 if self.prior_answers_dict().get(question_option_key) is not None:
-                    if isinstance(
-                        question_options := self.prior_answers_dict()
-                        .get(question_option_key)
-                        .answer,
-                        list,
-                    ):
-                        question_data["question_options"] = question_options
-                        self.question.question_options = question_options
+                    prior_question = self.prior_answers_dict().get(question_option_key)
+                    if hasattr(prior_question, "answer"):
+                        if isinstance(prior_question.answer, list):
+                            question_data["question_options"] = prior_question.answer
+                            self.question.question_options = prior_question.answer
+                    else:
+                        placeholder_options = [
+                            "N/A",
+                            "Will be populated by prior answer",
+                            "These are placeholder options",
+                        ]
+                        question_data["question_options"] = placeholder_options
+                        self.question.question_options = placeholder_options
+
+                    # if isinstance(
+                    #     question_options := self.prior_answers_dict()
+                    #     .get(question_option_key)
+                    #     .answer,
+                    #     list,
+                    # ):
+                    #     question_data["question_options"] = question_options
+                    #     self.question.question_options = question_options
 
             replacement_dict = (
                 {key: f"<see file {key}>" for key in self.scenario_file_keys}
@@ -257,9 +220,10 @@ class PromptConstructor:
             )
 
             if relevant_instructions != []:
-                preamble_text = Prompt(
-                    text="Before answer this question, you were given the following instructions: "
-                )
+                # preamble_text = Prompt(
+                #    text="You were given the following instructions: "
+                # )
+                preamble_text = Prompt(text="")
                 for instruction in relevant_instructions:
                     preamble_text += instruction.text
                 rendered_instructions = preamble_text + rendered_instructions

{edsl-0.1.35 → edsl-0.1.36}/edsl/conversation/Conversation.py

@@ -169,6 +169,7 @@ class Conversation:
             agent=speaker,
             just_answer=False,
             cache=self.cache,
+            model=speaker.model,
         )
         return results[0]
 
@@ -179,7 +180,6 @@ class Conversation:
         i = 0
         while await self.continue_conversation():
             speaker = self.next_speaker()
-            # breakpoint()
 
             next_statement = AgentStatement(
                 statement=await self.get_next_statement(

{edsl-0.1.35 → edsl-0.1.36}/edsl/coop/PriceFetcher.py

@@ -16,30 +16,26 @@ class PriceFetcher:
         if self._cached_prices is not None:
             return self._cached_prices
 
+        import os
         import requests
-        import csv
-        from io import StringIO
-
-        sheet_id = "1SAO3Bhntefl0XQHJv27rMxpvu6uzKDWNXFHRa7jrUDs"
-
-        # Construct the URL to fetch the CSV
-        url = f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv"
+        from edsl import CONFIG
 
         try:
-            # Fetch the CSV data
-            response = requests.get(url)
+            # Fetch the pricing data
+            url = f"{CONFIG.EXPECTED_PARROT_URL}/api/v0/prices"
+            api_key = os.getenv("EXPECTED_PARROT_API_KEY")
+            headers = {}
+            if api_key:
+                headers["Authorization"] = f"Bearer {api_key}"
+            else:
+                headers["Authorization"] = f"Bearer None"
+
+            response = requests.get(url, headers=headers, timeout=20)
             response.raise_for_status()  # Raise an exception for bad responses
 
-            # Parse the CSV data
-            csv_data = StringIO(response.text)
-            reader = csv.reader(csv_data)
-
-            # Convert to list of dictionaries
-            headers = next(reader)
-            data = [dict(zip(headers, row)) for row in reader]
+            # Parse the data
+            data = response.json()
 
-            # self._cached_prices = data
-            # return data
             price_lookup = {}
             for entry in data:
                 service = entry.get("service", None)

{edsl-0.1.35 → edsl-0.1.36}/edsl/coop/coop.py

@@ -6,6 +6,7 @@ from typing import Any, Optional, Union, Literal
 from uuid import UUID
 import edsl
 from edsl import CONFIG, CacheEntry, Jobs, Survey
+from edsl.exceptions.coop import CoopNoUUIDError, CoopServerResponseError
 from edsl.coop.utils import (
     EDSLObject,
     ObjectRegistry,
@@ -99,7 +100,7 @@ class Coop:
             if "Authorization" in message:
                 print(message)
                 message = "Please provide an Expected Parrot API key."
-            raise Exception(message)
+            raise CoopServerResponseError(message)
 
     def _json_handle_none(self, value: Any) -> Any:
         """
@@ -116,7 +117,7 @@ class Coop:
         Resolve the uuid from a uuid or a url.
         """
         if not url and not uuid:
-            raise Exception("No uuid or url provided for the object.")
+            raise CoopNoUUIDError("No uuid or url provided for the object.")
         if not uuid and url:
             uuid = url.split("/")[-1]
         return uuid
@@ -521,7 +522,7 @@ class Coop:
         self._resolve_server_response(response)
         response_json = response.json()
         return {
-            "uuid": response_json.get("jobs_uuid"),
+            "uuid": response_json.get("job_uuid"),
             "description": response_json.get("description"),
             "status": response_json.get("status"),
             "iterations": response_json.get("iterations"),
@@ -529,29 +530,41 @@ class Coop:
             "version": self._edsl_version,
         }
 
-    def remote_inference_get(self, job_uuid: str) -> dict:
+    def remote_inference_get(
+        self, job_uuid: Optional[str] = None, results_uuid: Optional[str] = None
+    ) -> dict:
         """
         Get the details of a remote inference job.
+        You can pass either the job uuid or the results uuid as a parameter.
+        If you pass both, the job uuid will be prioritized.
 
         :param job_uuid: The UUID of the EDSL job.
+        :param results_uuid: The UUID of the results associated with the EDSL job.
 
         >>> coop.remote_inference_get("9f8484ee-b407-40e4-9652-4133a7236c9c")
         {'jobs_uuid': '9f8484ee-b407-40e4-9652-4133a7236c9c', 'results_uuid': 'dd708234-31bf-4fe1-8747-6e232625e026', 'results_url': 'https://www.expectedparrot.com/content/dd708234-31bf-4fe1-8747-6e232625e026', 'status': 'completed', 'reason': None, 'price': 16, 'version': '0.1.29.dev4'}
         """
+        if job_uuid is None and results_uuid is None:
+            raise ValueError("Either job_uuid or results_uuid must be provided.")
+        elif job_uuid is not None:
+            params = {"job_uuid": job_uuid}
+        else:
+            params = {"results_uuid": results_uuid}
+
         response = self._send_server_request(
             uri="api/v0/remote-inference",
             method="GET",
-            params={"uuid": job_uuid},
+            params=params,
         )
         self._resolve_server_response(response)
         data = response.json()
         return {
-            "jobs_uuid": data.get("jobs_uuid"),
+            "job_uuid": data.get("job_uuid"),
             "results_uuid": data.get("results_uuid"),
             "results_url": f"{self.url}/content/{data.get('results_uuid')}",
             "status": data.get("status"),
             "reason": data.get("reason"),
-            "price": data.get("price"),
+            "credits_consumed": data.get("price"),
             "version": data.get("version"),
         }
 
@@ -584,7 +597,10 @@ class Coop:
         )
         self._resolve_server_response(response)
         response_json = response.json()
-        return response_json.get("cost")
+        return {
+            "credits": response_json.get("cost_in_credits"),
+            "usd": response_json.get("cost_in_usd"),
+        }
 
     ################
     # Remote Errors
@@ -649,6 +665,10 @@ class Coop:
         return response_json
 
     def fetch_prices(self) -> dict:
+        """
+        Fetch model prices from Coop. If the request fails, return an empty dict.
+        """
+
         from edsl.coop.PriceFetcher import PriceFetcher
 
         from edsl.config import CONFIG
@@ -659,6 +679,20 @@ class Coop:
         else:
             return {}
 
+    def fetch_rate_limit_config_vars(self) -> dict:
+        """
+        Fetch a dict of rate limit config vars from Coop.
+
+        The dict keys are RPM and TPM variables like EDSL_SERVICE_RPM_OPENAI.
+        """
+        response = self._send_server_request(
+            uri="api/v0/config-vars",
+            method="GET",
+        )
+        self._resolve_server_response(response)
+        data = response.json()
+        return data
+
 
 if __name__ == "__main__":
     sheet_data = fetch_sheet_data()

edsl-0.1.36/edsl/data/RemoteCacheSync.py

@@ -0,0 +1,97 @@
+class RemoteCacheSync:
+    def __init__(
+        self, coop, cache, output_func, remote_cache=True, remote_cache_description=""
+    ):
+        self.coop = coop
+        self.cache = cache
+        self._output = output_func
+        self.remote_cache = remote_cache
+        self.old_entry_keys = []
+        self.new_cache_entries = []
+        self.remote_cache_description = remote_cache_description
+
+    def __enter__(self):
+        if self.remote_cache:
+            self._sync_from_remote()
+            self.old_entry_keys = list(self.cache.keys())
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.remote_cache:
+            self._sync_to_remote()
+        return False  # Propagate exceptions
+
+    def _sync_from_remote(self):
+        cache_difference = self.coop.remote_cache_get_diff(self.cache.keys())
+        client_missing_cacheentries = cache_difference.get(
+            "client_missing_cacheentries", []
+        )
+        missing_entry_count = len(client_missing_cacheentries)
+
+        if missing_entry_count > 0:
+            self._output(
+                f"Updating local cache with {missing_entry_count:,} new "
+                f"{'entry' if missing_entry_count == 1 else 'entries'} from remote..."
+            )
+            self.cache.add_from_dict(
+                {entry.key: entry for entry in client_missing_cacheentries}
+            )
+            self._output("Local cache updated!")
+        else:
+            self._output("No new entries to add to local cache.")
+
+    def _sync_to_remote(self):
+        cache_difference = self.coop.remote_cache_get_diff(self.cache.keys())
+        server_missing_cacheentry_keys = cache_difference.get(
+            "server_missing_cacheentry_keys", []
+        )
+        server_missing_cacheentries = [
+            entry
+            for key in server_missing_cacheentry_keys
+            if (entry := self.cache.data.get(key)) is not None
+        ]
+
+        new_cache_entries = [
+            entry
+            for entry in self.cache.values()
+            if entry.key not in self.old_entry_keys
+        ]
+        server_missing_cacheentries.extend(new_cache_entries)
+        new_entry_count = len(server_missing_cacheentries)
+
+        if new_entry_count > 0:
+            self._output(
+                f"Updating remote cache with {new_entry_count:,} new "
+                f"{'entry' if new_entry_count == 1 else 'entries'}..."
+            )
+            self.coop.remote_cache_create_many(
+                server_missing_cacheentries,
+                visibility="private",
+                description=self.remote_cache_description,
+            )
+            self._output("Remote cache updated!")
+        else:
+            self._output("No new entries to add to remote cache.")
+
+        self._output(
+            f"There are {len(self.cache.keys()):,} entries in the local cache."
+        )
+
+
+# # Usage example
+# def run_job(self, n, progress_bar, cache, stop_on_exception, sidecar_model, print_exceptions, raise_validation_errors, use_remote_cache=True):
+#     with RemoteCacheSync(self.coop, cache, self._output, remote_cache=use_remote_cache):
+#         self._output("Running job...")
+#         results = self._run_local(
+#             n=n,
+#             progress_bar=progress_bar,
+#             cache=cache,
+#             stop_on_exception=stop_on_exception,
+#             sidecar_model=sidecar_model,
+#             print_exceptions=print_exceptions,
+#             raise_validation_errors=raise_validation_errors,
+#         )
+#         self._output("Job completed!")
+
+#     results.cache = cache.new_entries_cache()
+#     return results

edsl-0.1.36/edsl/exceptions/coop.py

@@ -0,0 +1,10 @@
+class CoopErrors(Exception):
+    pass
+
+
+class CoopNoUUIDError(CoopErrors):
+    pass
+
+
+class CoopServerResponseError(CoopErrors):
+    pass

{edsl-0.1.35 → edsl-0.1.36}/edsl/inference_services/InferenceServiceABC.py

@@ -1,6 +1,7 @@
 from abc import abstractmethod, ABC
 import os
 import re
+from datetime import datetime, timedelta
 from edsl.config import CONFIG
 
 
@@ -10,6 +11,8 @@ class InferenceServiceABC(ABC):
     Anthropic: https://docs.anthropic.com/en/api/rate-limits
     """
 
+    _coop_config_vars = None
+
     default_levels = {
         "google": {"tpm": 2_000_000, "rpm": 15},
         "openai": {"tpm": 2_000_000, "rpm": 10_000},
@@ -31,12 +34,37 @@ class InferenceServiceABC(ABC):
                 f"Class {cls.__name__} must have a 'model_exclude_list' attribute."
             )
 
+    @classmethod
+    def _should_refresh_coop_config_vars(cls):
+        """
+        Returns True if config vars have been fetched over 24 hours ago, and False otherwise.
+        """
+
+        if cls._last_config_fetch is None:
+            return True
+        return (datetime.now() - cls._last_config_fetch) > timedelta(hours=24)
+
     @classmethod
     def _get_limt(cls, limit_type: str) -> int:
         key = f"EDSL_SERVICE_{limit_type.upper()}_{cls._inference_service_.upper()}"
         if key in os.environ:
             return int(os.getenv(key))
 
+        if cls._coop_config_vars is None or cls._should_refresh_coop_config_vars():
+            try:
+                from edsl import Coop
+
+                c = Coop()
+                cls._coop_config_vars = c.fetch_rate_limit_config_vars()
+                cls._last_config_fetch = datetime.now()
+                if key in cls._coop_config_vars:
+                    return cls._coop_config_vars[key]
+            except Exception:
+                cls._coop_config_vars = None
+        else:
+            if key in cls._coop_config_vars:
+                return cls._coop_config_vars[key]
+
         if cls._inference_service_ in cls.default_levels:
             return int(cls.default_levels[cls._inference_service_][limit_type])
 

{edsl-0.1.35 → edsl-0.1.36}/edsl/inference_services/InferenceServicesCollection.py

@@ -56,13 +56,19 @@ class InferenceServicesCollection:
         self.services.append(service)
 
     def create_model_factory(self, model_name: str, service_name=None, index=None):
+        from edsl.inference_services.TestService import TestService
+
+        if model_name == "test":
+            return TestService.create_model(model_name)
+
+        if service_name:
+            for service in self.services:
+                if service_name == service._inference_service_:
+                    return service.create_model(model_name)
+
         for service in self.services:
             if model_name in self._get_service_available(service):
                 if service_name is None or service_name == service._inference_service_:
                     return service.create_model(model_name)
 
-        # if model_name == "test":
-        #     from edsl.language_models import LanguageModel
-        #     return LanguageModel(test = True)
-
         raise Exception(f"Model {model_name} not found in any of the services")