edsl 0.1.35__tar.gz → 0.1.36.dev1__tar.gz

{edsl-0.1.35 → edsl-0.1.36.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edsl
-Version: 0.1.35
+Version: 0.1.36.dev1
 Summary: Create and analyze LLM-based surveys
 Home-page: https://www.expectedparrot.com/
 License: MIT

edsl-0.1.36.dev1/edsl/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.36.dev1"

{edsl-0.1.35 → edsl-0.1.36.dev1}/edsl/agents/Agent.py

@@ -8,6 +8,9 @@ from typing import Callable, Optional, Union, Any
 from uuid import uuid4
 from edsl.Base import Base
 
+from edsl.prompts import Prompt
+from edsl.exceptions import QuestionScenarioRenderError
+
 from edsl.exceptions.agents import (
     AgentCombinationError,
     AgentDirectAnswerFunctionError,
@@ -44,7 +47,6 @@ class Agent(Base):
 
     def __init__(
         self,
-        # *,
         traits: Optional[dict] = None,
         name: Optional[str] = None,
         codebook: Optional[dict] = None,
@@ -112,6 +114,7 @@ class Agent(Base):
         self.instruction = instruction or self.default_instruction
         self.dynamic_traits_function = dynamic_traits_function
 
+        # Deal with dynamic traits function
         if self.dynamic_traits_function:
             self.dynamic_traits_function_name = self.dynamic_traits_function.__name__
             self.has_dynamic_traits_function = True
@@ -124,6 +127,7 @@ class Agent(Base):
                 dynamic_traits_function_name, dynamic_traits_function
             )
 
+        # Deal with direct answer function
         if answer_question_directly_source_code:
             self.answer_question_directly_function_name = (
                 answer_question_directly_function_name
@@ -140,10 +144,34 @@ class Agent(Base):
         self.current_question = None
 
         if traits_presentation_template is not None:
-            from edsl.prompts.library.agent_persona import AgentPersona
-
             self.traits_presentation_template = traits_presentation_template
-            self.agent_persona = AgentPersona(text=self.traits_presentation_template)
+        else:
+            self.traits_presentation_template = """Your traits: {{ traits }}"""
+
+    @property
+    def agent_persona(self) -> Prompt:
+        return Prompt(text=self.traits_presentation_template)
+
+    def prompt(self) -> str:
+        """Return the prompt for the agent.
+
+        Example usage:
+
+        >>> a = Agent(traits = {"age": 10, "hair": "brown", "height": 5.5})
+        >>> a.prompt()
+        Prompt(text=\"""Your traits: {'age': 10, 'hair': 'brown', 'height': 5.5}\""")
+        """
+        replacement_dict = (
+            self.traits | {"traits": self.traits} | {"codebook": self.codebook}
+        )
+        if undefined := self.agent_persona.undefined_template_variables(
+            replacement_dict
+        ):
+            raise QuestionScenarioRenderError(
+                f"Agent persona still has variables that were not rendered: {undefined}"
+            )
+        else:
+            return self.agent_persona.render(replacement_dict)
 
     def _check_dynamic_traits_function(self) -> None:
         """Check whether dynamic trait function is valid.
@@ -252,7 +280,6 @@ class Agent(Base):
             warnings.warn(
                 "Warning: overwriting existing answer_question_directly method"
             )
-            # print("Warning: overwriting existing answer_question_directly method")
 
         self.validate_response = validate_response
         self.translate_response = translate_response
@@ -575,9 +602,6 @@ class Agent(Base):
         if self.name == None:
             raw_data.pop("name")
 
-        import inspect
-
-        # print(raw_data)
         if hasattr(self, "dynamic_traits_function"):
             raw_data.pop(
                 "dynamic_traits_function", None
@@ -586,9 +610,9 @@ class Agent(Base):
             if dynamic_traits_func:
                 func = inspect.getsource(dynamic_traits_func)
                 raw_data["dynamic_traits_function_source_code"] = func
-                raw_data[
-                    "dynamic_traits_function_name"
-                ] = self.dynamic_traits_function_name
+                raw_data["dynamic_traits_function_name"] = (
+                    self.dynamic_traits_function_name
+                )
         if hasattr(self, "answer_question_directly"):
             raw_data.pop(
                 "answer_question_directly", None
@@ -604,9 +628,9 @@ class Agent(Base):
                 raw_data["answer_question_directly_source_code"] = inspect.getsource(
                     answer_question_directly_func
                 )
-                raw_data[
-                    "answer_question_directly_function_name"
-                ] = self.answer_question_directly_function_name
+                raw_data["answer_question_directly_function_name"] = (
+                    self.answer_question_directly_function_name
+                )
 
         return raw_data
 

{edsl-0.1.35 → edsl-0.1.36.dev1}/edsl/agents/Invigilator.py

@@ -4,7 +4,8 @@ from typing import Dict, Any, Optional
 
 from edsl.prompts.Prompt import Prompt
 from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
-from edsl.prompts.registry import get_classes as prompt_lookup
+
+# from edsl.prompts.registry import get_classes as prompt_lookup
 from edsl.exceptions.questions import QuestionAnswerValidationError
 from edsl.agents.InvigilatorBase import InvigilatorBase
 from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput

{edsl-0.1.35 → edsl-0.1.36.dev1}/edsl/agents/PromptConstructor.py

@@ -7,7 +7,8 @@ from jinja2 import Environment, meta
 
 from edsl.prompts.Prompt import Prompt
 from edsl.data_transfer_models import ImageInfo
-from edsl.prompts.registry import get_classes as prompt_lookup
+
+# from edsl.prompts.registry import get_classes as prompt_lookup
 from edsl.exceptions import QuestionScenarioRenderError
 
 from edsl.agents.prompt_helpers import PromptComponent, PromptList, PromptPlan
@@ -75,17 +76,8 @@ class PromptConstructor:
 
         if self.agent == Agent():  # if agent is empty, then return an empty prompt
             return Prompt(text="")
-        if not hasattr(self, "_agent_instructions_prompt"):
-            applicable_prompts = prompt_lookup(
-                component_type="agent_instructions",
-                model=self.model.model,
-            )
-            if len(applicable_prompts) == 0:
-                raise Exception("No applicable prompts found")
-            self._agent_instructions_prompt = applicable_prompts[0](
-                text=self.agent.instruction
-            )
-        return self._agent_instructions_prompt
+
+        return Prompt(text=self.agent.instruction)
 
     @property
     def agent_persona_prompt(self) -> Prompt:
@@ -93,51 +85,14 @@ class PromptConstructor:
         >>> from edsl.agents.InvigilatorBase import InvigilatorBase
         >>> i = InvigilatorBase.example()
         >>> i.prompt_constructor.agent_persona_prompt
-        Prompt(text=\"""You are an agent with the following persona:
-        {'age': 22, 'hair': 'brown', 'height': 5.5}\""")
-
+        Prompt(text=\"""Your traits: {'age': 22, 'hair': 'brown', 'height': 5.5}\""")
         """
         from edsl import Agent
 
-        if hasattr(self, "_agent_persona_prompt"):
-            return self._agent_persona_prompt
-
         if self.agent == Agent():  # if agent is empty, then return an empty prompt
             return Prompt(text="")
 
-        if not hasattr(self.agent, "agent_persona"):
-            applicable_prompts = prompt_lookup(
-                component_type="agent_persona",
-                model=self.model.model,
-            )
-            persona_prompt_template = applicable_prompts[0]()
-        else:
-            persona_prompt_template = self.agent.agent_persona
-
-        # TODO: This multiple passing of agent traits - not sure if it is necessary. Not harmful.
-        template_parameter_dictionary = (
-            self.agent.traits
-            | {"traits": self.agent.traits}
-            | {"codebook": self.agent.codebook}
-            | {"traits": self.agent.traits}
-        )
-
-        if undefined := persona_prompt_template.undefined_template_variables(
-            template_parameter_dictionary
-        ):
-            raise QuestionScenarioRenderError(
-                f"Agent persona still has variables that were not rendered: {undefined}"
-            )
-
-        persona_prompt = persona_prompt_template.render(template_parameter_dictionary)
-        if persona_prompt.has_variables:
-            raise QuestionScenarioRenderError(
-                "Agent persona still has variables that were not rendered."
-            )
-
-        self._agent_persona_prompt = persona_prompt
-
-        return self._agent_persona_prompt
+        return self.agent.prompt()
 
     def prior_answers_dict(self) -> dict:
         d = self.survey.question_names_to_questions()

{edsl-0.1.35 → edsl-0.1.36.dev1}/edsl/jobs/Jobs.py

@@ -180,17 +180,15 @@ class Jobs(Base):
                 scenario_indices.append(scenario_index)
                 models.append(invigilator.model.model)
                 question_names.append(invigilator.question.question_name)
-                # cost calculation
-                key = (invigilator.model._inference_service_, invigilator.model.model)
-                relevant_prices = price_lookup[key]
-                inverse_output_price = relevant_prices["output"]["one_usd_buys"]
-                inverse_input_price = relevant_prices["input"]["one_usd_buys"]
-                input_tokens = len(str(user_prompt) + str(system_prompt)) // 4
-                output_tokens = len(str(user_prompt) + str(system_prompt)) // 4
-                cost = input_tokens / float(
-                    inverse_input_price
-                ) + output_tokens / float(inverse_output_price)
-                costs.append(cost)
+
+                prompt_cost = self.estimate_prompt_cost(
+                    system_prompt=system_prompt,
+                    user_prompt=user_prompt,
+                    price_lookup=price_lookup,
+                    inference_service=invigilator.model._inference_service_,
+                    model=invigilator.model.model,
+                )
+                costs.append(prompt_cost["cost"])
 
         d = Dataset(
             [
@@ -214,50 +212,150 @@ class Jobs(Base):
         """Print the prompts."""
         self.prompts().to_scenario_list().print(format="rich")
 
-    def estimate_job_cost(self):
-        from edsl import Coop
+    @staticmethod
+    def estimate_prompt_cost(
+        system_prompt: str,
+        user_prompt: str,
+        price_lookup: dict,
+        inference_service: str,
+        model: str,
+    ):
+        """Estimates the cost of a prompt. Takes piping into account."""
 
-        c = Coop()
-        price_lookup = c.fetch_prices()
+        def get_piping_multiplier(prompt: str):
+            """Returns 2 if a prompt includes Jinja brances, and 1 otherwise."""
 
-        prompts = self.prompts()
+            if "{{" in prompt and "}}" in prompt:
+                return 2
+            return 1
 
-        text_len = 0
-        for prompt in prompts:
-            text_len += len(str(prompt))
+        # Look up prices per token
+        key = (inference_service, model)
+        relevant_prices = price_lookup[key]
+        output_price_per_token = 1 / float(relevant_prices["output"]["one_usd_buys"])
+        input_price_per_token = 1 / float(relevant_prices["input"]["one_usd_buys"])
 
-        input_token_aproximations = text_len // 4
+        # Compute the number of characters (double if the question involves piping)
+        user_prompt_chars = len(str(user_prompt)) * get_piping_multiplier(
+            str(user_prompt)
+        )
+        system_prompt_chars = len(str(system_prompt)) * get_piping_multiplier(
+            str(system_prompt)
+        )
 
-        aproximation_cost = {}
-        total_cost = 0
-        for model in self.models:
-            key = (model._inference_service_, model.model)
-            relevant_prices = price_lookup[key]
-            inverse_output_price = relevant_prices["output"]["one_usd_buys"]
-            inverse_input_price = relevant_prices["input"]["one_usd_buys"]
-
-            aproximation_cost[key] = {
-                "input": input_token_aproximations / float(inverse_input_price),
-                "output": input_token_aproximations / float(inverse_output_price),
-            }
-            ##TODO curenlty we approximate the number of output tokens with the number
-            # of input tokens. A better solution will be to compute the quesiton answer options length and sum them
-            # to compute the output tokens
-
-            total_cost += input_token_aproximations / float(inverse_input_price)
-            total_cost += input_token_aproximations / float(inverse_output_price)
-
-        # multiply_factor = len(self.agents or [1]) * len(self.scenarios or [1])
-        multiply_factor = 1
-        out = {
-            "input_token_aproximations": input_token_aproximations,
-            "models_costs": aproximation_cost,
-            "estimated_total_cost": total_cost * multiply_factor,
-            "multiply_factor": multiply_factor,
-            "single_config_cost": total_cost,
+        # Convert into tokens (1 token approx. equals 4 characters)
+        input_tokens = (user_prompt_chars + system_prompt_chars) // 4
+        output_tokens = input_tokens
+
+        cost = (
+            input_tokens * input_price_per_token
+            + output_tokens * output_price_per_token
+        )
+
+        return {
+            "input_tokens": input_tokens,
+            "output_tokens": output_tokens,
+            "cost": cost,
+        }
+
+    def estimate_job_cost_from_external_prices(self, price_lookup: dict):
+        """
+        Estimates the cost of a job according to the following assumptions:
+
+        - 1 token = 4 characters.
+        - Input tokens = output tokens.
+
+        price_lookup is an external pricing dictionary.
+        """
+
+        import pandas as pd
+
+        interviews = self.interviews()
+        data = []
+        for interview in interviews:
+            invigilators = [
+                interview._get_invigilator(question)
+                for question in self.survey.questions
+            ]
+            for invigilator in invigilators:
+                prompts = invigilator.get_prompts()
+
+                # By this point, agent and scenario data has already been added to the prompts
+                user_prompt = prompts["user_prompt"]
+                system_prompt = prompts["system_prompt"]
+                inference_service = invigilator.model._inference_service_
+                model = invigilator.model.model
+
+                prompt_cost = self.estimate_prompt_cost(
+                    system_prompt=system_prompt,
+                    user_prompt=user_prompt,
+                    price_lookup=price_lookup,
+                    inference_service=inference_service,
+                    model=model,
+                )
+
+                data.append(
+                    {
+                        "user_prompt": user_prompt,
+                        "system_prompt": system_prompt,
+                        "estimated_input_tokens": prompt_cost["input_tokens"],
+                        "estimated_output_tokens": prompt_cost["output_tokens"],
+                        "estimated_cost": prompt_cost["cost"],
+                        "inference_service": inference_service,
+                        "model": model,
+                    }
+                )
+
+        df = pd.DataFrame.from_records(data)
+
+        df = (
+            df.groupby(["inference_service", "model"])
+            .agg(
+                {
+                    "estimated_cost": "sum",
+                    "estimated_input_tokens": "sum",
+                    "estimated_output_tokens": "sum",
+                }
+            )
+            .reset_index()
+        )
+
+        estimated_costs_by_model = df.to_dict("records")
+
+        estimated_total_cost = sum(
+            model["estimated_cost"] for model in estimated_costs_by_model
+        )
+        estimated_total_input_tokens = sum(
+            model["estimated_input_tokens"] for model in estimated_costs_by_model
+        )
+        estimated_total_output_tokens = sum(
+            model["estimated_output_tokens"] for model in estimated_costs_by_model
+        )
+
+        output = {
+            "estimated_total_cost": estimated_total_cost,
+            "estimated_total_input_tokens": estimated_total_input_tokens,
+            "estimated_total_output_tokens": estimated_total_output_tokens,
+            "model_costs": estimated_costs_by_model,
         }
 
-        return out
+        return output
+
+    def estimate_job_cost(self):
+        """
+        Estimates the cost of a job according to the following assumptions:
+
+        - 1 token = 4 characters.
+        - Input tokens = output tokens.
+
+        Fetches prices from Coop.
+        """
+        from edsl import Coop
+
+        c = Coop()
+        price_lookup = c.fetch_prices()
+
+        return self.estimate_job_cost_from_external_prices(price_lookup=price_lookup)
 
     @staticmethod
     def _get_container_class(object):

{edsl-0.1.35 → edsl-0.1.36.dev1}/edsl/jobs/interviews/Interview.py

@@ -28,7 +28,7 @@ from edsl.jobs.interviews.InterviewExceptionCollection import (
     InterviewExceptionCollection,
 )
 
-from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
+# from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
 
 from edsl.surveys.base import EndOfSurvey
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
@@ -44,6 +44,10 @@ from edsl.agents.InvigilatorBase import InvigilatorBase
 
 from edsl.exceptions.language_models import LanguageModelNoResponseError
 
+from edsl.jobs.interviews.InterviewStatusLog import InterviewStatusLog
+from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
+from edsl.jobs.interviews.InterviewStatusDictionary import InterviewStatusDictionary
+
 
 from edsl import CONFIG
 
@@ -52,7 +56,7 @@ EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
 EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
 
 
-class Interview(InterviewStatusMixin):
+class Interview:
     """
     An 'interview' is one agent answering one survey, with one language model, for a given scenario.
 
@@ -100,21 +104,17 @@ class Interview(InterviewStatusMixin):
 
         """
         self.agent = agent
-        # what I would like to do
-        self.survey = copy.deepcopy(survey)  # survey  copy.deepcopy(survey)
-        # self.survey = survey
+        self.survey = copy.deepcopy(survey)
         self.scenario = scenario
         self.model = model
         self.debug = debug
         self.iteration = iteration
         self.cache = cache
-        self.answers: dict[
-            str, str
-        ] = Answers()  # will get filled in as interview progresses
+        self.answers: dict[str, str] = (
+            Answers()
+        )  # will get filled in as interview progresses
         self.sidecar_model = sidecar_model
 
-        # self.stop_on_exception = False
-
         # Trackers
         self.task_creators = TaskCreators()  # tracks the task creators
         self.exceptions = InterviewExceptionCollection()
@@ -131,6 +131,33 @@ class Interview(InterviewStatusMixin):
 
         self.failed_questions = []
 
+    @property
+    def has_exceptions(self) -> bool:
+        """Return True if there are exceptions."""
+        return len(self.exceptions) > 0
+
+    @property
+    def task_status_logs(self) -> InterviewStatusLog:
+        """Return the task status logs for the interview.
+
+        The keys are the question names; the values are the lists of status log changes for each task.
+        """
+        for task_creator in self.task_creators.values():
+            self._task_status_log_dict[task_creator.question.question_name] = (
+                task_creator.status_log
+            )
+        return self._task_status_log_dict
+
+    @property
+    def token_usage(self) -> InterviewTokenUsage:
+        """Determine how many tokens were used for the interview."""
+        return self.task_creators.token_usage
+
+    @property
+    def interview_status(self) -> InterviewStatusDictionary:
+        """Return a dictionary mapping task status codes to counts."""
+        return self.task_creators.interview_status
+
     # region: Serialization
     def _to_dict(self, include_exceptions=False) -> dict[str, Any]:
         """Return a dictionary representation of the Interview instance.
@@ -431,11 +458,11 @@ class Interview(InterviewStatusMixin):
         """
         current_question_index: int = self.to_index[current_question.question_name]
 
-        next_question: Union[
-            int, EndOfSurvey
-        ] = self.survey.rule_collection.next_question(
-            q_now=current_question_index,
-            answers=self.answers | self.scenario | self.agent["traits"],
+        next_question: Union[int, EndOfSurvey] = (
+            self.survey.rule_collection.next_question(
+                q_now=current_question_index,
+                answers=self.answers | self.scenario | self.agent["traits"],
+            )
         )
 
         next_question_index = next_question.next_q

{edsl-0.1.35 → edsl-0.1.36.dev1}/edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -1,8 +1,5 @@
 import traceback
 import datetime
-import time
-from collections import UserDict
-from edsl.jobs.FailedQuestion import FailedQuestion
 
 
 class InterviewExceptionEntry:

{edsl-0.1.35 → edsl-0.1.36.dev1}/edsl/jobs/tasks/QuestionTaskCreator.py

@@ -1,20 +1,16 @@
 import asyncio
 from typing import Callable, Union, List
 from collections import UserList, UserDict
-import time
 
 from edsl.jobs.buckets import ModelBuckets
 from edsl.exceptions import InterviewErrorPriorTaskCanceled
 
 from edsl.jobs.interviews.InterviewStatusDictionary import InterviewStatusDictionary
 from edsl.jobs.tasks.task_status_enum import TaskStatus, TaskStatusDescriptor
-
-# from edsl.jobs.tasks.task_management import TokensUsed
 from edsl.jobs.tasks.TaskStatusLog import TaskStatusLog
 from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
 from edsl.jobs.tokens.TokenUsage import TokenUsage
 from edsl.jobs.Answers import Answers
-
 from edsl.questions.QuestionBase import QuestionBase
 
 
@@ -130,7 +126,7 @@ class QuestionTaskCreator(UserList):
         await self.tokens_bucket.get_tokens(requested_tokens)
 
         if (estimated_wait_time := self.requests_bucket.wait_time(1)) > 0:
-            self.waiting = True
+            self.waiting = True  #  do we need this?
             self.task_status = TaskStatus.WAITING_FOR_REQUEST_CAPACITY
 
         await self.requests_bucket.get_tokens(1, cheat_bucket_capacity=True)

{edsl-0.1.35 → edsl-0.1.36.dev1}/edsl/language_models/LanguageModel.py

@@ -348,6 +348,9 @@ class LanguageModel(
         """
         # parameters = dict({})
 
+        # this is the case when data is loaded from a dict after serialization
+        if "parameters" in passed_parameter_dict:
+            passed_parameter_dict = passed_parameter_dict["parameters"]
         return {
             parameter_name: passed_parameter_dict.get(parameter_name, default_value)
             for parameter_name, default_value in default_parameter_dict.items()