edsl 0.1.33__py3-none-any.whl → 0.1.33.dev1__py3-none-any.whl

edsl/agents/InvigilatorBase.py

@@ -8,27 +8,34 @@ from edsl.data_transfer_models import AgentResponseDict
 
 from edsl.data.Cache import Cache
 
+# from edsl.agents.Agent import Agent
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.scenarios.Scenario import Scenario
 from edsl.surveys.MemoryPlan import MemoryPlan
 from edsl.language_models.LanguageModel import LanguageModel
 
-from edsl.data_transfer_models import EDSLResultObjectInput
-from edsl.agents.PromptConstructor import PromptConstructor
-
 
 class InvigilatorBase(ABC):
     """An invigiator (someone who administers an exam) is a class that is responsible for administering a question to an agent.
 
     >>> InvigilatorBase.example().answer_question()
-    {'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
+    {'message': '{"answer": "SPAM!"}'}
 
-    >>> InvigilatorBase.example().get_failed_task_result(failure_reason="Failed to get response").comment
-    'Failed to get response'
+    >>> InvigilatorBase.example().get_failed_task_result()
+    {'answer': None, 'comment': 'Failed to get response', ...
 
     This returns an empty prompt because there is no memory the agent needs to have at q0.
 
+    >>> InvigilatorBase.example().create_memory_prompt("q0")
+    Prompt(text=\"""\""")
 
+    >>> i = InvigilatorBase.example()
+    >>> i.current_answers = {"q0": "Prior answer"}
+    >>> i.memory_plan.add_single_memory("q1", "q0")
+    >>> i.create_memory_prompt("q1")
+    Prompt(text=\"""
+            Before the question you are now answering, you already answered the following question(s):
+    ...
     """
 
     def __init__(
@@ -44,7 +51,6 @@ class InvigilatorBase(ABC):
         iteration: Optional[int] = 1,
         additional_prompt_data: Optional[dict] = None,
         sidecar_model: Optional[LanguageModel] = None,
-        raise_validation_errors: Optional[bool] = True,
     ):
         """Initialize a new Invigilator."""
         self.agent = agent
@@ -58,78 +64,6 @@ class InvigilatorBase(ABC):
         self.cache = cache
         self.sidecar_model = sidecar_model
         self.survey = survey
-        self.raise_validation_errors = raise_validation_errors
-
-        self.raw_model_response = (
-            None  # placeholder for the raw response from the model
-        )
-
-    @property
-    def prompt_constructor(self) -> PromptConstructor:
-        """Return the prompt constructor."""
-        return PromptConstructor(self)
-
-    def to_dict(self):
-        attributes = [
-            "agent",
-            "question",
-            "scenario",
-            "model",
-            "memory_plan",
-            "current_answers",
-            "iteration",
-            "additional_prompt_data",
-            "cache",
-            "sidecar_model",
-            "survey",
-        ]
-
-        def serialize_attribute(attr):
-            value = getattr(self, attr)
-            if value is None:
-                return None
-            if hasattr(value, "to_dict"):
-                return value.to_dict()
-            if isinstance(value, (int, float, str, bool, dict, list)):
-                return value
-            return str(value)
-
-        return {attr: serialize_attribute(attr) for attr in attributes}
-
-    @classmethod
-    def from_dict(cls, data):
-        from edsl.agents.Agent import Agent
-        from edsl.questions import QuestionBase
-        from edsl.scenarios.Scenario import Scenario
-        from edsl.surveys.MemoryPlan import MemoryPlan
-        from edsl.language_models.LanguageModel import LanguageModel
-        from edsl.surveys.Survey import Survey
-
-        agent = Agent.from_dict(data["agent"])
-        question = QuestionBase.from_dict(data["question"])
-        scenario = Scenario.from_dict(data["scenario"])
-        model = LanguageModel.from_dict(data["model"])
-        memory_plan = MemoryPlan.from_dict(data["memory_plan"])
-        survey = Survey.from_dict(data["survey"])
-        current_answers = data["current_answers"]
-        iteration = data["iteration"]
-        additional_prompt_data = data["additional_prompt_data"]
-        cache = Cache.from_dict(data["cache"])
-        sidecar_model = LanguageModel.from_dict(data["sidecar_model"])
-
-        return cls(
-            agent=agent,
-            question=question,
-            scenario=scenario,
-            model=model,
-            memory_plan=memory_plan,
-            current_answers=current_answers,
-            survey=survey,
-            iteration=iteration,
-            additional_prompt_data=additional_prompt_data,
-            cache=cache,
-            sidecar_model=sidecar_model,
-        )
 
     def __repr__(self) -> str:
         """Return a string representation of the Invigilator.
@@ -140,45 +74,18 @@ class InvigilatorBase(ABC):
         """
         return f"{self.__class__.__name__}(agent={repr(self.agent)}, question={repr(self.question)}, scneario={repr(self.scenario)}, model={repr(self.model)}, memory_plan={repr(self.memory_plan)}, current_answers={repr(self.current_answers)}, iteration{repr(self.iteration)}, additional_prompt_data={repr(self.additional_prompt_data)}, cache={repr(self.cache)}, sidecarmodel={repr(self.sidecar_model)})"
 
-    def get_failed_task_result(self, failure_reason) -> EDSLResultObjectInput:
+    def get_failed_task_result(self) -> AgentResponseDict:
         """Return an AgentResponseDict used in case the question-asking fails.
 
-        Possible reasons include:
-        - Legimately skipped because of skip logic
-        - Failed to get response from the model
-
+        >>> InvigilatorBase.example().get_failed_task_result()
+        {'answer': None, 'comment': 'Failed to get response', ...}
         """
-        data = {
-            "answer": None,
-            "generated_tokens": None,
-            "comment": failure_reason,
-            "question_name": self.question.question_name,
-            "prompts": self.get_prompts(),
-            "cached_response": None,
-            "raw_model_response": None,
-            "cache_used": None,
-            "cache_key": None,
-        }
-        return EDSLResultObjectInput(**data)
-
-        # breakpoint()
-        # if hasattr(self, "augmented_model_response"):
-        #     import json
-
-        #     generated_tokens = json.loads(self.augmented_model_response)["answer"][
-        #         "generated_tokens"
-        #     ]
-        # else:
-        #     generated_tokens = "Filled in by InvigilatorBase.get_failed_task_result"
-        # agent_response_dict = AgentResponseDict(
-        #     answer=None,
-        #     comment="Failed to get usable response",
-        #     generated_tokens=generated_tokens,
-        #     question_name=self.question.question_name,
-        #     prompts=self.get_prompts(),
-        # )
-        # # breakpoint()
-        # return agent_response_dict
+        return AgentResponseDict(
+            answer=None,
+            comment="Failed to get response",
+            question_name=self.question.question_name,
+            prompts=self.get_prompts(),
+        )
 
     def get_prompts(self) -> Dict[str, Prompt]:
         """Return the prompt used."""
@@ -204,10 +111,24 @@ class InvigilatorBase(ABC):
 
         return main()
 
+    def create_memory_prompt(self, question_name: str) -> Prompt:
+        """Create a memory for the agent.
+
+        The returns a memory prompt for the agent.
+
+        >>> i = InvigilatorBase.example()
+        >>> i.current_answers = {"q0": "Prior answer"}
+        >>> i.memory_plan.add_single_memory("q1", "q0")
+        >>> p = i.create_memory_prompt("q1")
+        >>> p.text.strip().replace("\\n", " ").replace("\\t", " ")
+        'Before the question you are now answering, you already answered the following question(s):          Question: Do you like school?  Answer: Prior answer'
+        """
+        return self.memory_plan.get_memory_prompt_fragment(
+            question_name, self.current_answers
+        )
+
     @classmethod
-    def example(
-        cls, throw_an_exception=False, question=None, scenario=None, survey=None
-    ) -> "InvigilatorBase":
+    def example(cls, throw_an_exception=False, question=None, scenario=None):
         """Return an example invigilator.
 
         >>> InvigilatorBase.example()
@@ -222,53 +143,43 @@ class InvigilatorBase(ABC):
 
         from edsl.enums import InferenceServiceType
 
-        from edsl import Model
-
-        model = Model("test", canned_response="SPAM!")
-        # class TestLanguageModelGood(LanguageModel):
-        #     """A test language model."""
+        class TestLanguageModelGood(LanguageModel):
+            """A test language model."""
 
-        #     _model_ = "test"
-        #     _parameters_ = {"temperature": 0.5}
-        #     _inference_service_ = InferenceServiceType.TEST.value
+            _model_ = "test"
+            _parameters_ = {"temperature": 0.5}
+            _inference_service_ = InferenceServiceType.TEST.value
 
-        #     async def async_execute_model_call(
-        #         self, user_prompt: str, system_prompt: str
-        #     ) -> dict[str, Any]:
-        #         await asyncio.sleep(0.1)
-        #         if hasattr(self, "throw_an_exception"):
-        #             raise Exception("Error!")
-        #         return {"message": """{"answer": "SPAM!"}"""}
+            async def async_execute_model_call(
+                self, user_prompt: str, system_prompt: str
+            ) -> dict[str, Any]:
+                await asyncio.sleep(0.1)
+                if hasattr(self, "throw_an_exception"):
+                    raise Exception("Error!")
+                return {"message": """{"answer": "SPAM!"}"""}
 
-        #     def parse_response(self, raw_response: dict[str, Any]) -> str:
-        #         """Parse the response from the model."""
-        #         return raw_response["message"]
+            def parse_response(self, raw_response: dict[str, Any]) -> str:
+                """Parse the response from the model."""
+                return raw_response["message"]
 
+        model = TestLanguageModelGood()
         if throw_an_exception:
             model.throw_an_exception = True
         agent = Agent.example()
         # question = QuestionMultipleChoice.example()
         from edsl.surveys import Survey
 
-        if not survey:
-            survey = Survey.example()
-        # if question:
-        # need to have the focal question name in the list of names
-        # survey._questions[0].question_name = question.question_name
-        #    survey.add_question(question)
-        if question:
-            survey.add_question(question)
-
+        survey = Survey.example()
         question = question or survey.questions[0]
         scenario = scenario or Scenario.example()
         # memory_plan = None #memory_plan = MemoryPlan()
         from edsl import Survey
 
-        memory_plan = MemoryPlan(survey=survey)
+        memory_plan = MemoryPlan(survey=Survey.example())
         current_answers = None
-        from edsl.agents.PromptConstructor import PromptConstructor
+        from edsl.agents.PromptConstructionMixin import PromptConstructorMixin
 
-        class InvigilatorExample(InvigilatorBase):
+        class InvigilatorExample(PromptConstructorMixin, InvigilatorBase):
             """An example invigilator."""
 
             async def async_answer_question(self):

edsl/agents/{PromptConstructor.py → PromptConstructionMixin.py}

@@ -1,15 +1,15 @@
-from __future__ import annotations
-from typing import Dict, Any, Optional, Set
+from typing import Dict, Any, Optional
 from collections import UserList
-import enum
-
-from jinja2 import Environment, meta
 
+# from functools import reduce
 from edsl.prompts.Prompt import Prompt
-from edsl.data_transfer_models import ImageInfo
+
+# from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
 from edsl.prompts.registry import get_classes as prompt_lookup
 from edsl.exceptions import QuestionScenarioRenderError
 
+import enum
+
 
 class PromptComponent(enum.Enum):
     AGENT_INSTRUCTIONS = "agent_instructions"
@@ -18,21 +18,6 @@ class PromptComponent(enum.Enum):
     PRIOR_QUESTION_MEMORY = "prior_question_memory"
 
 
-def get_jinja2_variables(template_str: str) -> Set[str]:
-    """
-    Extracts all variable names from a Jinja2 template using Jinja2's built-in parsing.
-
-    Args:
-    template_str (str): The Jinja2 template string
-
-    Returns:
-    Set[str]: A set of variable names found in the template
-    """
-    env = Environment()
-    ast = env.parse(template_str)
-    return meta.find_undeclared_variables(ast)
-
-
 class PromptList(UserList):
     separator = Prompt(" ")
 
@@ -151,7 +136,7 @@ class PromptPlan:
         }
 
 
-class PromptConstructor:
+class PromptConstructorMixin:
     """Mixin for constructing prompts for the LLM call.
 
     The pieces of a prompt are:
@@ -163,40 +148,16 @@ class PromptConstructor:
     This is mixed into the Invigilator class.
     """
 
-    def __init__(self, invigilator):
-        self.invigilator = invigilator
-        self.agent = invigilator.agent
-        self.question = invigilator.question
-        self.scenario = invigilator.scenario
-        self.survey = invigilator.survey
-        self.model = invigilator.model
-        self.current_answers = invigilator.current_answers
-        self.memory_plan = invigilator.memory_plan
-        self.prompt_plan = PromptPlan()  # Assuming PromptPlan is defined elsewhere
-
-        # prompt_plan = PromptPlan()
-
-    @property
-    def scenario_image_keys(self):
-        image_entries = []
-
-        for key, value in self.scenario.items():
-            if isinstance(value, ImageInfo):
-                image_entries.append(key)
-        return image_entries
+    prompt_plan = PromptPlan()
 
     @property
     def agent_instructions_prompt(self) -> Prompt:
         """
         >>> from edsl.agents.InvigilatorBase import InvigilatorBase
         >>> i = InvigilatorBase.example()
-        >>> i.prompt_constructor.agent_instructions_prompt
+        >>> i.agent_instructions_prompt
         Prompt(text=\"""You are answering questions as if you were a human. Do not break character.\""")
         """
-        from edsl import Agent
-
-        if self.agent == Agent():  # if agent is empty, then return an empty prompt
-            return Prompt(text="")
         if not hasattr(self, "_agent_instructions_prompt"):
             applicable_prompts = prompt_lookup(
                 component_type="agent_instructions",
@@ -214,17 +175,12 @@ class PromptConstructor:
         """
         >>> from edsl.agents.InvigilatorBase import InvigilatorBase
         >>> i = InvigilatorBase.example()
-        >>> i.prompt_constructor.agent_persona_prompt
+        >>> i.agent_persona_prompt
         Prompt(text=\"""You are an agent with the following persona:
         {'age': 22, 'hair': 'brown', 'height': 5.5}\""")
 
         """
         if not hasattr(self, "_agent_persona_prompt"):
-            from edsl import Agent
-
-            if self.agent == Agent():  # if agent is empty, then return an empty prompt
-                return Prompt(text="")
-
             if not hasattr(self.agent, "agent_persona"):
                 applicable_prompts = prompt_lookup(
                     component_type="agent_persona",
@@ -269,69 +225,92 @@ class PromptConstructor:
                     d[new_question].comment = answer
         return d
 
-    @property
-    def question_image_keys(self):
-        raw_question_text = self.question.question_text
-        variables = get_jinja2_variables(raw_question_text)
-        question_image_keys = []
-        for var in variables:
-            if var in self.scenario_image_keys:
-                question_image_keys.append(var)
-        return question_image_keys
-
     @property
     def question_instructions_prompt(self) -> Prompt:
         """
         >>> from edsl.agents.InvigilatorBase import InvigilatorBase
         >>> i = InvigilatorBase.example()
-        >>> i.prompt_constructor.question_instructions_prompt
-        Prompt(text=\"""...
+        >>> i.question_instructions_prompt
+        Prompt(text=\"""You are being asked the following question: Do you like school?
+        The options are
+        <BLANKLINE>
+        0: yes
+        <BLANKLINE>
+        1: no
+        <BLANKLINE>
+        Return a valid JSON formatted like this, selecting only the number of the option:
+        {"answer": <put answer code here>, "comment": "<put explanation here>"}
+        Only 1 option may be selected.\""")
+
+        >>> from edsl import QuestionFreeText
+        >>> q = QuestionFreeText(question_text = "Consider {{ X }}. What is your favorite color?", question_name = "q_color")
+        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
+        >>> i = InvigilatorBase.example(question = q)
+        >>> i.question_instructions_prompt
+        Traceback (most recent call last):
         ...
+        edsl.exceptions.questions.QuestionScenarioRenderError: Question instructions still has variables: ['X'].
+
+
+        >>> from edsl import QuestionFreeText
+        >>> q = QuestionFreeText(question_text = "You were asked the question '{{ q0.question_text }}'. What is your favorite color?", question_name = "q_color")
+        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
+        >>> i = InvigilatorBase.example(question = q)
+        >>> i.question_instructions_prompt
+        Prompt(text=\"""You are being asked the following question: You were asked the question 'Do you like school?'. What is your favorite color?
+        Return a valid JSON formatted like this:
+        {"answer": "<put free text answer here>"}\""")
+
+        >>> from edsl import QuestionFreeText
+        >>> q = QuestionFreeText(question_text = "You stated '{{ q0.answer }}'. What is your favorite color?", question_name = "q_color")
+        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
+        >>> i = InvigilatorBase.example(question = q)
+        >>> i.current_answers = {"q0": "I like school"}
+        >>> i.question_instructions_prompt
+        Prompt(text=\"""You are being asked the following question: You stated 'I like school'. What is your favorite color?
+        Return a valid JSON formatted like this:
+        {"answer": "<put free text answer here>"}\""")
+
+
         """
         if not hasattr(self, "_question_instructions_prompt"):
             question_prompt = self.question.get_instructions(model=self.model.model)
 
-            # Are any of the scenario values ImageInfo
+            # TODO: Try to populate the answers in the question object if they are available
+            # d = self.survey.question_names_to_questions()
+            # for question, answer in self.current_answers.items():
+            #     if question in d:
+            #         d[question].answer = answer
+            #     else:
+            #         # adds a comment to the question
+            #         if (new_question := question.split("_comment")[0]) in d:
+            #             d[new_question].comment = answer
 
             question_data = self.question.data.copy()
 
             # check to see if the question_options is actually a string
             # This is used when the user is using the question_options as a variable from a sceario
-            # if "question_options" in question_data:
-            if isinstance(self.question.data.get("question_options", None), str):
-                env = Environment()
-                parsed_content = env.parse(self.question.data["question_options"])
-                question_option_key = list(
-                    meta.find_undeclared_variables(parsed_content)
-                )[0]
-
-                if isinstance(
-                    question_options := self.scenario.get(question_option_key), list
-                ):
-                    question_data["question_options"] = question_options
-                    self.question.question_options = question_options
-
-            replacement_dict = (
-                {key: "<see image>" for key in self.scenario_image_keys}
-                | question_data
-                | {
-                    k: v
-                    for k, v in self.scenario.items()
-                    if k not in self.scenario_image_keys
-                }  # don't include images in the replacement dict
+            if "question_options" in question_data:
+                if isinstance(self.question.data["question_options"], str):
+                    from jinja2 import Environment, meta
+
+                    env = Environment()
+                    parsed_content = env.parse(self.question.data["question_options"])
+                    question_option_key = list(
+                        meta.find_undeclared_variables(parsed_content)
+                    )[0]
+                    question_data["question_options"] = self.scenario.get(
+                        question_option_key
+                    )
+
+            # breakpoint()
+            rendered_instructions = question_prompt.render(
+                question_data
+                | self.scenario
                 | self.prior_answers_dict()
                 | {"agent": self.agent}
-                | {
-                    "use_code": getattr(self.question, "_use_code", True),
-                    "include_comment": getattr(
-                        self.question, "_include_comment", False
-                    ),
-                }
             )
 
-            rendered_instructions = question_prompt.render(replacement_dict)
-
-            # is there anything left to render?
             undefined_template_variables = (
                 rendered_instructions.undefined_template_variables({})
             )
@@ -345,25 +324,11 @@ class PromptConstructor:
                     )
 
             if undefined_template_variables:
+                print(undefined_template_variables)
                 raise QuestionScenarioRenderError(
                     f"Question instructions still has variables: {undefined_template_variables}."
                 )
 
-            ####################################
-            # Check if question has instructions - these are instructions in a Survey that can apply to multiple follow-on questions
-            ####################################
-            relevant_instructions = self.survey.relevant_instructions(
-                self.question.question_name
-            )
-
-            if relevant_instructions != []:
-                preamble_text = Prompt(
-                    text="Before answer this question, you were given the following instructions: "
-                )
-                for instruction in relevant_instructions:
-                    preamble_text += instruction.text
-                rendered_instructions = preamble_text + rendered_instructions
-
             self._question_instructions_prompt = rendered_instructions
         return self._question_instructions_prompt
 
@@ -380,23 +345,6 @@ class PromptConstructor:
             self._prior_question_memory_prompt = memory_prompt
         return self._prior_question_memory_prompt
 
-    def create_memory_prompt(self, question_name: str) -> Prompt:
-        """Create a memory for the agent.
-
-        The returns a memory prompt for the agent.
-
-        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> i = InvigilatorBase.example()
-        >>> i.current_answers = {"q0": "Prior answer"}
-        >>> i.memory_plan.add_single_memory("q1", "q0")
-        >>> p = i.prompt_constructor.create_memory_prompt("q1")
-        >>> p.text.strip().replace("\\n", " ").replace("\\t", " ")
-        'Before the question you are now answering, you already answered the following question(s):          Question: Do you like school?  Answer: Prior answer'
-        """
-        return self.memory_plan.get_memory_prompt_fragment(
-            question_name, self.current_answers
-        )
-
     def construct_system_prompt(self) -> Prompt:
         """Construct the system prompt for the LLM call."""
         import warnings
@@ -420,10 +368,17 @@ class PromptConstructor:
 
         >>> from edsl import QuestionFreeText
         >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> q = QuestionFreeText(question_text="How are you today?", question_name="q_new")
+        >>> q = QuestionFreeText(question_text="How are you today?", question_name="q0")
         >>> i = InvigilatorBase.example(question = q)
         >>> i.get_prompts()
         {'user_prompt': ..., 'system_prompt': ...}
+        >>> scenario = i._get_scenario_with_image()
+        >>> scenario.has_image
+        True
+        >>> q = QuestionFreeText(question_text="How are you today?", question_name="q0")
+        >>> i = InvigilatorBase.example(question = q, scenario = scenario)
+        >>> i.get_prompts()
+        {'user_prompt': ..., 'system_prompt': ..., 'encoded_image': ...'}
         """
         prompts = self.prompt_plan.get_prompts(
             agent_instructions=self.agent_instructions_prompt,
@@ -431,16 +386,12 @@ class PromptConstructor:
             question_instructions=self.question_instructions_prompt,
             prior_question_memory=self.prior_question_memory_prompt,
         )
-        if len(self.question_image_keys) > 1:
-            raise ValueError("We can only handle one image per question.")
-        elif len(self.question_image_keys) == 1:
-            prompts["encoded_image"] = self.scenario[
-                self.question_image_keys[0]
-            ].encoded_image
 
+        if hasattr(self.scenario, "has_image") and self.scenario.has_image:
+            prompts["encoded_image"] = self.scenario["encoded_image"]
         return prompts
 
-    def _get_scenario_with_image(self) -> Scenario:
+    def _get_scenario_with_image(self) -> Dict[str, Any]:
         """This is a helper function to get a scenario with an image, for testing purposes."""
         from edsl import Scenario
 

edsl/agents/__init__.py

@@ -1,3 +1,2 @@
 from edsl.agents.Agent import Agent
 from edsl.agents.AgentList import AgentList
-from edsl.agents.InvigilatorBase import InvigilatorBase