edsl 0.1.31.dev4__py3-none-any.whl → 0.1.33__py3-none-any.whl

edsl/agents/InvigilatorBase.py

@@ -8,34 +8,27 @@ from edsl.data_transfer_models import AgentResponseDict
 
 from edsl.data.Cache import Cache
 
-# from edsl.agents.Agent import Agent
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.scenarios.Scenario import Scenario
 from edsl.surveys.MemoryPlan import MemoryPlan
 from edsl.language_models.LanguageModel import LanguageModel
 
+from edsl.data_transfer_models import EDSLResultObjectInput
+from edsl.agents.PromptConstructor import PromptConstructor
+
 
 class InvigilatorBase(ABC):
     """An invigiator (someone who administers an exam) is a class that is responsible for administering a question to an agent.
 
     >>> InvigilatorBase.example().answer_question()
-    {'message': '{"answer": "SPAM!"}'}
+    {'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
 
-    >>> InvigilatorBase.example().get_failed_task_result()
-    {'answer': None, 'comment': 'Failed to get response', ...
+    >>> InvigilatorBase.example().get_failed_task_result(failure_reason="Failed to get response").comment
+    'Failed to get response'
 
     This returns an empty prompt because there is no memory the agent needs to have at q0.
 
-    >>> InvigilatorBase.example().create_memory_prompt("q0")
-    Prompt(text=\"""\""")
 
-    >>> i = InvigilatorBase.example()
-    >>> i.current_answers = {"q0": "Prior answer"}
-    >>> i.memory_plan.add_single_memory("q1", "q0")
-    >>> i.create_memory_prompt("q1")
-    Prompt(text=\"""
-            Before the question you are now answering, you already answered the following question(s):
-    ...
     """
 
     def __init__(
@@ -51,6 +44,7 @@ class InvigilatorBase(ABC):
         iteration: Optional[int] = 1,
         additional_prompt_data: Optional[dict] = None,
         sidecar_model: Optional[LanguageModel] = None,
+        raise_validation_errors: Optional[bool] = True,
     ):
         """Initialize a new Invigilator."""
         self.agent = agent
@@ -64,6 +58,78 @@ class InvigilatorBase(ABC):
         self.cache = cache
         self.sidecar_model = sidecar_model
         self.survey = survey
+        self.raise_validation_errors = raise_validation_errors
+
+        self.raw_model_response = (
+            None  # placeholder for the raw response from the model
+        )
+
+    @property
+    def prompt_constructor(self) -> PromptConstructor:
+        """Return the prompt constructor."""
+        return PromptConstructor(self)
+
+    def to_dict(self):
+        attributes = [
+            "agent",
+            "question",
+            "scenario",
+            "model",
+            "memory_plan",
+            "current_answers",
+            "iteration",
+            "additional_prompt_data",
+            "cache",
+            "sidecar_model",
+            "survey",
+        ]
+
+        def serialize_attribute(attr):
+            value = getattr(self, attr)
+            if value is None:
+                return None
+            if hasattr(value, "to_dict"):
+                return value.to_dict()
+            if isinstance(value, (int, float, str, bool, dict, list)):
+                return value
+            return str(value)
+
+        return {attr: serialize_attribute(attr) for attr in attributes}
+
+    @classmethod
+    def from_dict(cls, data):
+        from edsl.agents.Agent import Agent
+        from edsl.questions import QuestionBase
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.surveys.MemoryPlan import MemoryPlan
+        from edsl.language_models.LanguageModel import LanguageModel
+        from edsl.surveys.Survey import Survey
+
+        agent = Agent.from_dict(data["agent"])
+        question = QuestionBase.from_dict(data["question"])
+        scenario = Scenario.from_dict(data["scenario"])
+        model = LanguageModel.from_dict(data["model"])
+        memory_plan = MemoryPlan.from_dict(data["memory_plan"])
+        survey = Survey.from_dict(data["survey"])
+        current_answers = data["current_answers"]
+        iteration = data["iteration"]
+        additional_prompt_data = data["additional_prompt_data"]
+        cache = Cache.from_dict(data["cache"])
+        sidecar_model = LanguageModel.from_dict(data["sidecar_model"])
+
+        return cls(
+            agent=agent,
+            question=question,
+            scenario=scenario,
+            model=model,
+            memory_plan=memory_plan,
+            current_answers=current_answers,
+            survey=survey,
+            iteration=iteration,
+            additional_prompt_data=additional_prompt_data,
+            cache=cache,
+            sidecar_model=sidecar_model,
+        )
 
     def __repr__(self) -> str:
         """Return a string representation of the Invigilator.
@@ -74,18 +140,45 @@ class InvigilatorBase(ABC):
         """
         return f"{self.__class__.__name__}(agent={repr(self.agent)}, question={repr(self.question)}, scneario={repr(self.scenario)}, model={repr(self.model)}, memory_plan={repr(self.memory_plan)}, current_answers={repr(self.current_answers)}, iteration{repr(self.iteration)}, additional_prompt_data={repr(self.additional_prompt_data)}, cache={repr(self.cache)}, sidecarmodel={repr(self.sidecar_model)})"
 
-    def get_failed_task_result(self) -> AgentResponseDict:
+    def get_failed_task_result(self, failure_reason) -> EDSLResultObjectInput:
         """Return an AgentResponseDict used in case the question-asking fails.
 
-        >>> InvigilatorBase.example().get_failed_task_result()
-        {'answer': None, 'comment': 'Failed to get response', ...}
+        Possible reasons include:
+        - Legimately skipped because of skip logic
+        - Failed to get response from the model
+
         """
-        return AgentResponseDict(
-            answer=None,
-            comment="Failed to get response",
-            question_name=self.question.question_name,
-            prompts=self.get_prompts(),
-        )
+        data = {
+            "answer": None,
+            "generated_tokens": None,
+            "comment": failure_reason,
+            "question_name": self.question.question_name,
+            "prompts": self.get_prompts(),
+            "cached_response": None,
+            "raw_model_response": None,
+            "cache_used": None,
+            "cache_key": None,
+        }
+        return EDSLResultObjectInput(**data)
+
+        # breakpoint()
+        # if hasattr(self, "augmented_model_response"):
+        #     import json
+
+        #     generated_tokens = json.loads(self.augmented_model_response)["answer"][
+        #         "generated_tokens"
+        #     ]
+        # else:
+        #     generated_tokens = "Filled in by InvigilatorBase.get_failed_task_result"
+        # agent_response_dict = AgentResponseDict(
+        #     answer=None,
+        #     comment="Failed to get usable response",
+        #     generated_tokens=generated_tokens,
+        #     question_name=self.question.question_name,
+        #     prompts=self.get_prompts(),
+        # )
+        # # breakpoint()
+        # return agent_response_dict
 
     def get_prompts(self) -> Dict[str, Prompt]:
         """Return the prompt used."""
@@ -111,24 +204,10 @@ class InvigilatorBase(ABC):
 
         return main()
 
-    def create_memory_prompt(self, question_name: str) -> Prompt:
-        """Create a memory for the agent.
-
-        The returns a memory prompt for the agent.
-
-        >>> i = InvigilatorBase.example()
-        >>> i.current_answers = {"q0": "Prior answer"}
-        >>> i.memory_plan.add_single_memory("q1", "q0")
-        >>> p = i.create_memory_prompt("q1")
-        >>> p.text.strip().replace("\\n", " ").replace("\\t", " ")
-        'Before the question you are now answering, you already answered the following question(s):          Question: Do you like school?  Answer: Prior answer'
-        """
-        return self.memory_plan.get_memory_prompt_fragment(
-            question_name, self.current_answers
-        )
-
     @classmethod
-    def example(cls, throw_an_exception=False, question=None, scenario=None):
+    def example(
+        cls, throw_an_exception=False, question=None, scenario=None, survey=None
+    ) -> "InvigilatorBase":
         """Return an example invigilator.
 
         >>> InvigilatorBase.example()
@@ -143,43 +222,53 @@ class InvigilatorBase(ABC):
 
         from edsl.enums import InferenceServiceType
 
-        class TestLanguageModelGood(LanguageModel):
-            """A test language model."""
+        from edsl import Model
+
+        model = Model("test", canned_response="SPAM!")
+        # class TestLanguageModelGood(LanguageModel):
+        #     """A test language model."""
 
-            _model_ = "test"
-            _parameters_ = {"temperature": 0.5}
-            _inference_service_ = InferenceServiceType.TEST.value
+        #     _model_ = "test"
+        #     _parameters_ = {"temperature": 0.5}
+        #     _inference_service_ = InferenceServiceType.TEST.value
 
-            async def async_execute_model_call(
-                self, user_prompt: str, system_prompt: str
-            ) -> dict[str, Any]:
-                await asyncio.sleep(0.1)
-                if hasattr(self, "throw_an_exception"):
-                    raise Exception("Error!")
-                return {"message": """{"answer": "SPAM!"}"""}
+        #     async def async_execute_model_call(
+        #         self, user_prompt: str, system_prompt: str
+        #     ) -> dict[str, Any]:
+        #         await asyncio.sleep(0.1)
+        #         if hasattr(self, "throw_an_exception"):
+        #             raise Exception("Error!")
+        #         return {"message": """{"answer": "SPAM!"}"""}
 
-            def parse_response(self, raw_response: dict[str, Any]) -> str:
-                """Parse the response from the model."""
-                return raw_response["message"]
+        #     def parse_response(self, raw_response: dict[str, Any]) -> str:
+        #         """Parse the response from the model."""
+        #         return raw_response["message"]
 
-        model = TestLanguageModelGood()
         if throw_an_exception:
             model.throw_an_exception = True
         agent = Agent.example()
         # question = QuestionMultipleChoice.example()
         from edsl.surveys import Survey
 
-        survey = Survey.example()
+        if not survey:
+            survey = Survey.example()
+        # if question:
+        # need to have the focal question name in the list of names
+        # survey._questions[0].question_name = question.question_name
+        #    survey.add_question(question)
+        if question:
+            survey.add_question(question)
+
         question = question or survey.questions[0]
         scenario = scenario or Scenario.example()
         # memory_plan = None #memory_plan = MemoryPlan()
         from edsl import Survey
 
-        memory_plan = MemoryPlan(survey=Survey.example())
+        memory_plan = MemoryPlan(survey=survey)
         current_answers = None
-        from edsl.agents.PromptConstructionMixin import PromptConstructorMixin
+        from edsl.agents.PromptConstructor import PromptConstructor
 
-        class InvigilatorExample(PromptConstructorMixin, InvigilatorBase):
+        class InvigilatorExample(InvigilatorBase):
             """An example invigilator."""
 
             async def async_answer_question(self):

edsl/agents/{PromptConstructionMixin.py → PromptConstructor.py}

@@ -1,15 +1,15 @@
-from typing import Dict, Any, Optional
+from __future__ import annotations
+from typing import Dict, Any, Optional, Set
 from collections import UserList
+import enum
 
-# from functools import reduce
-from edsl.prompts.Prompt import Prompt
+from jinja2 import Environment, meta
 
-# from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
+from edsl.prompts.Prompt import Prompt
+from edsl.data_transfer_models import ImageInfo
 from edsl.prompts.registry import get_classes as prompt_lookup
 from edsl.exceptions import QuestionScenarioRenderError
 
-import enum
-
 
 class PromptComponent(enum.Enum):
     AGENT_INSTRUCTIONS = "agent_instructions"
@@ -18,6 +18,21 @@ class PromptComponent(enum.Enum):
     PRIOR_QUESTION_MEMORY = "prior_question_memory"
 
 
+def get_jinja2_variables(template_str: str) -> Set[str]:
+    """
+    Extracts all variable names from a Jinja2 template using Jinja2's built-in parsing.
+
+    Args:
+    template_str (str): The Jinja2 template string
+
+    Returns:
+    Set[str]: A set of variable names found in the template
+    """
+    env = Environment()
+    ast = env.parse(template_str)
+    return meta.find_undeclared_variables(ast)
+
+
 class PromptList(UserList):
     separator = Prompt(" ")
 
@@ -136,7 +151,7 @@ class PromptPlan:
         }
 
 
-class PromptConstructorMixin:
+class PromptConstructor:
     """Mixin for constructing prompts for the LLM call.
 
     The pieces of a prompt are:
@@ -148,16 +163,40 @@ class PromptConstructorMixin:
     This is mixed into the Invigilator class.
     """
 
-    prompt_plan = PromptPlan()
+    def __init__(self, invigilator):
+        self.invigilator = invigilator
+        self.agent = invigilator.agent
+        self.question = invigilator.question
+        self.scenario = invigilator.scenario
+        self.survey = invigilator.survey
+        self.model = invigilator.model
+        self.current_answers = invigilator.current_answers
+        self.memory_plan = invigilator.memory_plan
+        self.prompt_plan = PromptPlan()  # Assuming PromptPlan is defined elsewhere
+
+        # prompt_plan = PromptPlan()
+
+    @property
+    def scenario_image_keys(self):
+        image_entries = []
+
+        for key, value in self.scenario.items():
+            if isinstance(value, ImageInfo):
+                image_entries.append(key)
+        return image_entries
 
     @property
     def agent_instructions_prompt(self) -> Prompt:
         """
         >>> from edsl.agents.InvigilatorBase import InvigilatorBase
         >>> i = InvigilatorBase.example()
-        >>> i.agent_instructions_prompt
+        >>> i.prompt_constructor.agent_instructions_prompt
         Prompt(text=\"""You are answering questions as if you were a human. Do not break character.\""")
         """
+        from edsl import Agent
+
+        if self.agent == Agent():  # if agent is empty, then return an empty prompt
+            return Prompt(text="")
         if not hasattr(self, "_agent_instructions_prompt"):
             applicable_prompts = prompt_lookup(
                 component_type="agent_instructions",
@@ -175,12 +214,17 @@ class PromptConstructorMixin:
         """
         >>> from edsl.agents.InvigilatorBase import InvigilatorBase
         >>> i = InvigilatorBase.example()
-        >>> i.agent_persona_prompt
+        >>> i.prompt_constructor.agent_persona_prompt
         Prompt(text=\"""You are an agent with the following persona:
         {'age': 22, 'hair': 'brown', 'height': 5.5}\""")
 
         """
         if not hasattr(self, "_agent_persona_prompt"):
+            from edsl import Agent
+
+            if self.agent == Agent():  # if agent is empty, then return an empty prompt
+                return Prompt(text="")
+
             if not hasattr(self.agent, "agent_persona"):
                 applicable_prompts = prompt_lookup(
                     component_type="agent_persona",
@@ -214,83 +258,80 @@ class PromptConstructorMixin:
 
         return self._agent_persona_prompt
 
+    def prior_answers_dict(self) -> dict:
+        d = self.survey.question_names_to_questions()
+        for question, answer in self.current_answers.items():
+            if question in d:
+                d[question].answer = answer
+            else:
+                # adds a comment to the question
+                if (new_question := question.split("_comment")[0]) in d:
+                    d[new_question].comment = answer
+        return d
+
+    @property
+    def question_image_keys(self):
+        raw_question_text = self.question.question_text
+        variables = get_jinja2_variables(raw_question_text)
+        question_image_keys = []
+        for var in variables:
+            if var in self.scenario_image_keys:
+                question_image_keys.append(var)
+        return question_image_keys
+
     @property
     def question_instructions_prompt(self) -> Prompt:
         """
         >>> from edsl.agents.InvigilatorBase import InvigilatorBase
         >>> i = InvigilatorBase.example()
-        >>> i.question_instructions_prompt
-        Prompt(text=\"""You are being asked the following question: Do you like school?
-        The options are
-        <BLANKLINE>
-        0: yes
-        <BLANKLINE>
-        1: no
-        <BLANKLINE>
-        Return a valid JSON formatted like this, selecting only the number of the option:
-        {"answer": <put answer code here>, "comment": "<put explanation here>"}
-        Only 1 option may be selected.\""")
-
-        >>> from edsl import QuestionFreeText
-        >>> q = QuestionFreeText(question_text = "Consider {{ X }}. What is your favorite color?", question_name = "q_color")
-        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> i = InvigilatorBase.example(question = q)
-        >>> i.question_instructions_prompt
-        Traceback (most recent call last):
+        >>> i.prompt_constructor.question_instructions_prompt
+        Prompt(text=\"""...
         ...
-        edsl.exceptions.questions.QuestionScenarioRenderError: Question instructions still has variables: ['X'].
-
-
-        >>> from edsl import QuestionFreeText
-        >>> q = QuestionFreeText(question_text = "You were asked the question '{{ q0.question_text }}'. What is your favorite color?", question_name = "q_color")
-        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> i = InvigilatorBase.example(question = q)
-        >>> i.question_instructions_prompt
-        Prompt(text=\"""You are being asked the following question: You were asked the question 'Do you like school?'. What is your favorite color?
-        Return a valid JSON formatted like this:
-        {"answer": "<put free text answer here>"}\""")
-
-        >>> from edsl import QuestionFreeText
-        >>> q = QuestionFreeText(question_text = "You stated '{{ q0.answer }}'. What is your favorite color?", question_name = "q_color")
-        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> i = InvigilatorBase.example(question = q)
-        >>> i.current_answers = {"q0": "I like school"}
-        >>> i.question_instructions_prompt
-        Prompt(text=\"""You are being asked the following question: You stated 'I like school'. What is your favorite color?
-        Return a valid JSON formatted like this:
-        {"answer": "<put free text answer here>"}\""")
-
-
         """
         if not hasattr(self, "_question_instructions_prompt"):
             question_prompt = self.question.get_instructions(model=self.model.model)
 
-            # TODO: Try to populate the answers in the question object if they are available
-            d = self.survey.question_names_to_questions()
-            for question, answer in self.current_answers.items():
-                if question in d:
-                    d[question].answer = answer
-                else:
-                    # adds a comment to the question
-                    if (new_question := question.split("_comment")[0]) in d:
-                        d[new_question].comment = answer
+            # Are any of the scenario values ImageInfo
 
             question_data = self.question.data.copy()
 
-            # check to see if the questio_options is actuall a string
-            if "question_options" in question_data:
-                if isinstance(self.question.data["question_options"], str):
-                    from jinja2 import Environment, meta
-                    env = Environment()
-                    parsed_content = env.parse(self.question.data['question_options'])
-                    question_option_key = list(meta.find_undeclared_variables(parsed_content))[0]
-                    question_data["question_options"] = self.scenario.get(question_option_key)   
-
-            #breakpoint()        
-            rendered_instructions = question_prompt.render(
-                question_data | self.scenario | d | {"agent": self.agent}
+            # check to see if the question_options is actually a string
+            # This is used when the user is using the question_options as a variable from a sceario
+            # if "question_options" in question_data:
+            if isinstance(self.question.data.get("question_options", None), str):
+                env = Environment()
+                parsed_content = env.parse(self.question.data["question_options"])
+                question_option_key = list(
+                    meta.find_undeclared_variables(parsed_content)
+                )[0]
+
+                if isinstance(
+                    question_options := self.scenario.get(question_option_key), list
+                ):
+                    question_data["question_options"] = question_options
+                    self.question.question_options = question_options
+
+            replacement_dict = (
+                {key: "<see image>" for key in self.scenario_image_keys}
+                | question_data
+                | {
+                    k: v
+                    for k, v in self.scenario.items()
+                    if k not in self.scenario_image_keys
+                }  # don't include images in the replacement dict
+                | self.prior_answers_dict()
+                | {"agent": self.agent}
+                | {
+                    "use_code": getattr(self.question, "_use_code", True),
+                    "include_comment": getattr(
+                        self.question, "_include_comment", False
+                    ),
+                }
             )
 
+            rendered_instructions = question_prompt.render(replacement_dict)
+
+            # is there anything left to render?
             undefined_template_variables = (
                 rendered_instructions.undefined_template_variables({})
             )
@@ -304,11 +345,25 @@ class PromptConstructorMixin:
                     )
 
             if undefined_template_variables:
-                print(undefined_template_variables)
                 raise QuestionScenarioRenderError(
                     f"Question instructions still has variables: {undefined_template_variables}."
                 )
 
+            ####################################
+            # Check if question has instructions - these are instructions in a Survey that can apply to multiple follow-on questions
+            ####################################
+            relevant_instructions = self.survey.relevant_instructions(
+                self.question.question_name
+            )
+
+            if relevant_instructions != []:
+                preamble_text = Prompt(
+                    text="Before answer this question, you were given the following instructions: "
+                )
+                for instruction in relevant_instructions:
+                    preamble_text += instruction.text
+                rendered_instructions = preamble_text + rendered_instructions
+
             self._question_instructions_prompt = rendered_instructions
         return self._question_instructions_prompt
 
@@ -321,10 +376,27 @@ class PromptConstructorMixin:
             if self.memory_plan is not None:
                 memory_prompt += self.create_memory_prompt(
                     self.question.question_name
-                ).render(self.scenario)
+                ).render(self.scenario | self.prior_answers_dict())
             self._prior_question_memory_prompt = memory_prompt
         return self._prior_question_memory_prompt
 
+    def create_memory_prompt(self, question_name: str) -> Prompt:
+        """Create a memory for the agent.
+
+        The returns a memory prompt for the agent.
+
+        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
+        >>> i = InvigilatorBase.example()
+        >>> i.current_answers = {"q0": "Prior answer"}
+        >>> i.memory_plan.add_single_memory("q1", "q0")
+        >>> p = i.prompt_constructor.create_memory_prompt("q1")
+        >>> p.text.strip().replace("\\n", " ").replace("\\t", " ")
+        'Before the question you are now answering, you already answered the following question(s):          Question: Do you like school?  Answer: Prior answer'
+        """
+        return self.memory_plan.get_memory_prompt_fragment(
+            question_name, self.current_answers
+        )
+
     def construct_system_prompt(self) -> Prompt:
         """Construct the system prompt for the LLM call."""
         import warnings
@@ -348,17 +420,10 @@ class PromptConstructorMixin:
 
         >>> from edsl import QuestionFreeText
         >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> q = QuestionFreeText(question_text="How are you today?", question_name="q0")
+        >>> q = QuestionFreeText(question_text="How are you today?", question_name="q_new")
         >>> i = InvigilatorBase.example(question = q)
         >>> i.get_prompts()
         {'user_prompt': ..., 'system_prompt': ...}
-        >>> scenario = i._get_scenario_with_image()
-        >>> scenario.has_image
-        True
-        >>> q = QuestionFreeText(question_text="How are you today?", question_name="q0")
-        >>> i = InvigilatorBase.example(question = q, scenario = scenario)
-        >>> i.get_prompts()
-        {'user_prompt': ..., 'system_prompt': ..., 'encoded_image': ...'}
         """
         prompts = self.prompt_plan.get_prompts(
             agent_instructions=self.agent_instructions_prompt,
@@ -366,12 +431,16 @@ class PromptConstructorMixin:
             question_instructions=self.question_instructions_prompt,
             prior_question_memory=self.prior_question_memory_prompt,
         )
+        if len(self.question_image_keys) > 1:
+            raise ValueError("We can only handle one image per question.")
+        elif len(self.question_image_keys) == 1:
+            prompts["encoded_image"] = self.scenario[
+                self.question_image_keys[0]
+            ].encoded_image
 
-        if hasattr(self.scenario, "has_image") and self.scenario.has_image:
-            prompts["encoded_image"] = self.scenario["encoded_image"]
         return prompts
 
-    def _get_scenario_with_image(self) -> Dict[str, Any]:
+    def _get_scenario_with_image(self) -> Scenario:
         """This is a helper function to get a scenario with an image, for testing purposes."""
         from edsl import Scenario
 

edsl/agents/__init__.py

@@ -1,2 +1,3 @@
 from edsl.agents.Agent import Agent
 from edsl.agents.AgentList import AgentList
+from edsl.agents.InvigilatorBase import InvigilatorBase