edsl 0.1.33.dev1__py3-none-any.whl → 0.1.33.dev2__py3-none-any.whl

edsl/agents/InvigilatorBase.py

@@ -8,21 +8,22 @@ from edsl.data_transfer_models import AgentResponseDict
 
 from edsl.data.Cache import Cache
 
-# from edsl.agents.Agent import Agent
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.scenarios.Scenario import Scenario
 from edsl.surveys.MemoryPlan import MemoryPlan
 from edsl.language_models.LanguageModel import LanguageModel
 
+from edsl.data_transfer_models import EDSLResultObjectInput
+
 
 class InvigilatorBase(ABC):
     """An invigiator (someone who administers an exam) is a class that is responsible for administering a question to an agent.
 
     >>> InvigilatorBase.example().answer_question()
-    {'message': '{"answer": "SPAM!"}'}
+    {'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
 
-    >>> InvigilatorBase.example().get_failed_task_result()
-    {'answer': None, 'comment': 'Failed to get response', ...
+    >>> InvigilatorBase.example().get_failed_task_result(failure_reason="Failed to get response").comment
+    'Failed to get response'
 
     This returns an empty prompt because there is no memory the agent needs to have at q0.
 
@@ -51,6 +52,7 @@ class InvigilatorBase(ABC):
         iteration: Optional[int] = 1,
         additional_prompt_data: Optional[dict] = None,
         sidecar_model: Optional[LanguageModel] = None,
+        raise_validation_errors: Optional[bool] = True,
     ):
         """Initialize a new Invigilator."""
         self.agent = agent
@@ -64,6 +66,73 @@ class InvigilatorBase(ABC):
         self.cache = cache
         self.sidecar_model = sidecar_model
         self.survey = survey
+        self.raise_validation_errors = raise_validation_errors
+
+        self.raw_model_response = (
+            None  # placeholder for the raw response from the model
+        )
+
+    def to_dict(self):
+        attributes = [
+            "agent",
+            "question",
+            "scenario",
+            "model",
+            "memory_plan",
+            "current_answers",
+            "iteration",
+            "additional_prompt_data",
+            "cache",
+            "sidecar_model",
+            "survey",
+        ]
+
+        def serialize_attribute(attr):
+            value = getattr(self, attr)
+            if value is None:
+                return None
+            if hasattr(value, "to_dict"):
+                return value.to_dict()
+            if isinstance(value, (int, float, str, bool, dict, list)):
+                return value
+            return str(value)
+
+        return {attr: serialize_attribute(attr) for attr in attributes}
+
+    @classmethod
+    def from_dict(cls, data):
+        from edsl.agents.Agent import Agent
+        from edsl.questions import QuestionBase
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.surveys.MemoryPlan import MemoryPlan
+        from edsl.language_models.LanguageModel import LanguageModel
+        from edsl.surveys.Survey import Survey
+
+        agent = Agent.from_dict(data["agent"])
+        question = QuestionBase.from_dict(data["question"])
+        scenario = Scenario.from_dict(data["scenario"])
+        model = LanguageModel.from_dict(data["model"])
+        memory_plan = MemoryPlan.from_dict(data["memory_plan"])
+        survey = Survey.from_dict(data["survey"])
+        current_answers = data["current_answers"]
+        iteration = data["iteration"]
+        additional_prompt_data = data["additional_prompt_data"]
+        cache = Cache.from_dict(data["cache"])
+        sidecar_model = LanguageModel.from_dict(data["sidecar_model"])
+
+        return cls(
+            agent=agent,
+            question=question,
+            scenario=scenario,
+            model=model,
+            memory_plan=memory_plan,
+            current_answers=current_answers,
+            survey=survey,
+            iteration=iteration,
+            additional_prompt_data=additional_prompt_data,
+            cache=cache,
+            sidecar_model=sidecar_model,
+        )
 
     def __repr__(self) -> str:
         """Return a string representation of the Invigilator.
@@ -74,18 +143,45 @@ class InvigilatorBase(ABC):
         """
         return f"{self.__class__.__name__}(agent={repr(self.agent)}, question={repr(self.question)}, scneario={repr(self.scenario)}, model={repr(self.model)}, memory_plan={repr(self.memory_plan)}, current_answers={repr(self.current_answers)}, iteration{repr(self.iteration)}, additional_prompt_data={repr(self.additional_prompt_data)}, cache={repr(self.cache)}, sidecarmodel={repr(self.sidecar_model)})"
 
-    def get_failed_task_result(self) -> AgentResponseDict:
+    def get_failed_task_result(self, failure_reason) -> EDSLResultObjectInput:
         """Return an AgentResponseDict used in case the question-asking fails.
 
-        >>> InvigilatorBase.example().get_failed_task_result()
-        {'answer': None, 'comment': 'Failed to get response', ...}
+        Possible reasons include:
+        - Legimately skipped because of skip logic
+        - Failed to get response from the model
+
         """
-        return AgentResponseDict(
-            answer=None,
-            comment="Failed to get response",
-            question_name=self.question.question_name,
-            prompts=self.get_prompts(),
-        )
+        data = {
+            "answer": None,
+            "generated_tokens": None,
+            "comment": failure_reason,
+            "question_name": self.question.question_name,
+            "prompts": self.get_prompts(),
+            "cached_response": None,
+            "raw_model_response": None,
+            "cache_used": None,
+            "cache_key": None,
+        }
+        return EDSLResultObjectInput(**data)
+
+        # breakpoint()
+        # if hasattr(self, "augmented_model_response"):
+        #     import json
+
+        #     generated_tokens = json.loads(self.augmented_model_response)["answer"][
+        #         "generated_tokens"
+        #     ]
+        # else:
+        #     generated_tokens = "Filled in by InvigilatorBase.get_failed_task_result"
+        # agent_response_dict = AgentResponseDict(
+        #     answer=None,
+        #     comment="Failed to get usable response",
+        #     generated_tokens=generated_tokens,
+        #     question_name=self.question.question_name,
+        #     prompts=self.get_prompts(),
+        # )
+        # # breakpoint()
+        # return agent_response_dict
 
     def get_prompts(self) -> Dict[str, Prompt]:
         """Return the prompt used."""
@@ -128,7 +224,9 @@ class InvigilatorBase(ABC):
         )
 
     @classmethod
-    def example(cls, throw_an_exception=False, question=None, scenario=None):
+    def example(
+        cls, throw_an_exception=False, question=None, scenario=None, survey=None
+    ) -> "InvigilatorBase":
         """Return an example invigilator.
 
         >>> InvigilatorBase.example()
@@ -143,39 +241,49 @@ class InvigilatorBase(ABC):
 
         from edsl.enums import InferenceServiceType
 
-        class TestLanguageModelGood(LanguageModel):
-            """A test language model."""
+        from edsl import Model
 
-            _model_ = "test"
-            _parameters_ = {"temperature": 0.5}
-            _inference_service_ = InferenceServiceType.TEST.value
+        model = Model("test", canned_response="SPAM!")
+        # class TestLanguageModelGood(LanguageModel):
+        #     """A test language model."""
 
-            async def async_execute_model_call(
-                self, user_prompt: str, system_prompt: str
-            ) -> dict[str, Any]:
-                await asyncio.sleep(0.1)
-                if hasattr(self, "throw_an_exception"):
-                    raise Exception("Error!")
-                return {"message": """{"answer": "SPAM!"}"""}
+        #     _model_ = "test"
+        #     _parameters_ = {"temperature": 0.5}
+        #     _inference_service_ = InferenceServiceType.TEST.value
 
-            def parse_response(self, raw_response: dict[str, Any]) -> str:
-                """Parse the response from the model."""
-                return raw_response["message"]
+        #     async def async_execute_model_call(
+        #         self, user_prompt: str, system_prompt: str
+        #     ) -> dict[str, Any]:
+        #         await asyncio.sleep(0.1)
+        #         if hasattr(self, "throw_an_exception"):
+        #             raise Exception("Error!")
+        #         return {"message": """{"answer": "SPAM!"}"""}
+
+        #     def parse_response(self, raw_response: dict[str, Any]) -> str:
+        #         """Parse the response from the model."""
+        #         return raw_response["message"]
 
-        model = TestLanguageModelGood()
         if throw_an_exception:
             model.throw_an_exception = True
         agent = Agent.example()
         # question = QuestionMultipleChoice.example()
         from edsl.surveys import Survey
 
-        survey = Survey.example()
+        if not survey:
+            survey = Survey.example()
+        # if question:
+        # need to have the focal question name in the list of names
+        # survey._questions[0].question_name = question.question_name
+        #    survey.add_question(question)
+        if question:
+            survey.add_question(question)
+
         question = question or survey.questions[0]
         scenario = scenario or Scenario.example()
         # memory_plan = None #memory_plan = MemoryPlan()
         from edsl import Survey
 
-        memory_plan = MemoryPlan(survey=Survey.example())
+        memory_plan = MemoryPlan(survey=survey)
         current_answers = None
         from edsl.agents.PromptConstructionMixin import PromptConstructorMixin
 

edsl/agents/PromptConstructionMixin.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 from typing import Dict, Any, Optional
 from collections import UserList
 
@@ -231,47 +232,8 @@ class PromptConstructorMixin:
         >>> from edsl.agents.InvigilatorBase import InvigilatorBase
         >>> i = InvigilatorBase.example()
         >>> i.question_instructions_prompt
-        Prompt(text=\"""You are being asked the following question: Do you like school?
-        The options are
-        <BLANKLINE>
-        0: yes
-        <BLANKLINE>
-        1: no
-        <BLANKLINE>
-        Return a valid JSON formatted like this, selecting only the number of the option:
-        {"answer": <put answer code here>, "comment": "<put explanation here>"}
-        Only 1 option may be selected.\""")
-
-        >>> from edsl import QuestionFreeText
-        >>> q = QuestionFreeText(question_text = "Consider {{ X }}. What is your favorite color?", question_name = "q_color")
-        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> i = InvigilatorBase.example(question = q)
-        >>> i.question_instructions_prompt
-        Traceback (most recent call last):
+        Prompt(text=\"""...
         ...
-        edsl.exceptions.questions.QuestionScenarioRenderError: Question instructions still has variables: ['X'].
-
-
-        >>> from edsl import QuestionFreeText
-        >>> q = QuestionFreeText(question_text = "You were asked the question '{{ q0.question_text }}'. What is your favorite color?", question_name = "q_color")
-        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> i = InvigilatorBase.example(question = q)
-        >>> i.question_instructions_prompt
-        Prompt(text=\"""You are being asked the following question: You were asked the question 'Do you like school?'. What is your favorite color?
-        Return a valid JSON formatted like this:
-        {"answer": "<put free text answer here>"}\""")
-
-        >>> from edsl import QuestionFreeText
-        >>> q = QuestionFreeText(question_text = "You stated '{{ q0.answer }}'. What is your favorite color?", question_name = "q_color")
-        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> i = InvigilatorBase.example(question = q)
-        >>> i.current_answers = {"q0": "I like school"}
-        >>> i.question_instructions_prompt
-        Prompt(text=\"""You are being asked the following question: You stated 'I like school'. What is your favorite color?
-        Return a valid JSON formatted like this:
-        {"answer": "<put free text answer here>"}\""")
-
-
         """
         if not hasattr(self, "_question_instructions_prompt"):
             question_prompt = self.question.get_instructions(model=self.model.model)
@@ -290,27 +252,37 @@ class PromptConstructorMixin:
 
             # check to see if the question_options is actually a string
             # This is used when the user is using the question_options as a variable from a sceario
-            if "question_options" in question_data:
-                if isinstance(self.question.data["question_options"], str):
-                    from jinja2 import Environment, meta
-
-                    env = Environment()
-                    parsed_content = env.parse(self.question.data["question_options"])
-                    question_option_key = list(
-                        meta.find_undeclared_variables(parsed_content)
-                    )[0]
-                    question_data["question_options"] = self.scenario.get(
-                        question_option_key
-                    )
-
-            # breakpoint()
-            rendered_instructions = question_prompt.render(
+            # if "question_options" in question_data:
+            if isinstance(self.question.data.get("question_options", None), str):
+                from jinja2 import Environment, meta
+
+                env = Environment()
+                parsed_content = env.parse(self.question.data["question_options"])
+                question_option_key = list(
+                    meta.find_undeclared_variables(parsed_content)
+                )[0]
+
+                if isinstance(
+                    question_options := self.scenario.get(question_option_key), list
+                ):
+                    question_data["question_options"] = question_options
+                    self.question.question_options = question_options
+
+            replacement_dict = (
                 question_data
                 | self.scenario
                 | self.prior_answers_dict()
                 | {"agent": self.agent}
+                | {
+                    "use_code": getattr(self.question, "_use_code", True),
+                    "include_comment": getattr(
+                        self.question, "_include_comment", False
+                    ),
+                }
             )
-
+            # breakpoint()
+            rendered_instructions = question_prompt.render(replacement_dict)
+            # breakpoint()
             undefined_template_variables = (
                 rendered_instructions.undefined_template_variables({})
             )
@@ -324,11 +296,23 @@ class PromptConstructorMixin:
                     )
 
             if undefined_template_variables:
-                print(undefined_template_variables)
                 raise QuestionScenarioRenderError(
                     f"Question instructions still has variables: {undefined_template_variables}."
                 )
 
+            # Check if question has an instructions
+            relevant_instructions = self.survey.relevant_instructions(
+                self.question.question_name
+            )
+
+            if relevant_instructions != []:
+                preamble_text = Prompt(
+                    text="Before answer this question, you were given the following instructions: "
+                )
+                for instruction in relevant_instructions:
+                    preamble_text += instruction.text
+                rendered_instructions = preamble_text + rendered_instructions
+
             self._question_instructions_prompt = rendered_instructions
         return self._question_instructions_prompt
 
@@ -368,17 +352,10 @@ class PromptConstructorMixin:
 
         >>> from edsl import QuestionFreeText
         >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> q = QuestionFreeText(question_text="How are you today?", question_name="q0")
+        >>> q = QuestionFreeText(question_text="How are you today?", question_name="q_new")
         >>> i = InvigilatorBase.example(question = q)
         >>> i.get_prompts()
         {'user_prompt': ..., 'system_prompt': ...}
-        >>> scenario = i._get_scenario_with_image()
-        >>> scenario.has_image
-        True
-        >>> q = QuestionFreeText(question_text="How are you today?", question_name="q0")
-        >>> i = InvigilatorBase.example(question = q, scenario = scenario)
-        >>> i.get_prompts()
-        {'user_prompt': ..., 'system_prompt': ..., 'encoded_image': ...'}
         """
         prompts = self.prompt_plan.get_prompts(
             agent_instructions=self.agent_instructions_prompt,
@@ -391,7 +368,7 @@ class PromptConstructorMixin:
             prompts["encoded_image"] = self.scenario["encoded_image"]
         return prompts
 
-    def _get_scenario_with_image(self) -> Dict[str, Any]:
+    def _get_scenario_with_image(self) -> Scenario:
         """This is a helper function to get a scenario with an image, for testing purposes."""
         from edsl import Scenario
 

edsl/agents/__init__.py

@@ -1,2 +1,3 @@
 from edsl.agents.Agent import Agent
 from edsl.agents.AgentList import AgentList
+from edsl.agents.InvigilatorBase import InvigilatorBase

edsl/auto/AutoStudy.py

@@ -0,0 +1,117 @@
+from typing import Optional
+
+from edsl import Model
+from edsl.auto.StageQuestions import StageQuestions
+from edsl.auto.StagePersona import StagePersona
+from edsl.auto.StagePersonaDimensions import StagePersonaDimensions
+from edsl.auto.StagePersonaDimensionValues import StagePersonaDimensionValues
+from edsl.auto.StagePersonaDimensionValueRanges import (
+    StagePersonaDimensionValueRanges,
+)
+from edsl.auto.StageLabelQuestions import StageLabelQuestions
+from edsl.auto.StageGenerateSurvey import StageGenerateSurvey
+
+# from edsl.auto.StageBase import gen_pipeline
+
+from edsl.auto.utilities import agent_generator, create_agents, gen_pipeline
+
+
+class AutoStudy:
+    def __init__(
+        self,
+        overall_question: str,
+        population: str,
+        model: Optional["Model"] = None,
+        survey: Optional["Survey"] = None,
+        agent_list: Optional["AgentList"] = None,
+        default_num_agents=11,
+    ):
+        self.overall_question = overall_question
+        self.population = population
+        self._survey = survey
+        self._agent_list = agent_list
+        self._agent_list_generator = None
+        self._persona_mapping = None
+        self._results = None
+        self.default_num_agents = default_num_agents
+        self.model = model or Model()
+
+    @property
+    def survey(self):
+        if self._survey is None:
+            self._survey = self._create_survey()
+        return self._survey
+
+    @property
+    def persona_mapping(self):
+        if self._persona_mapping is None:
+            self._persona_mapping = self._create_persona_mapping()
+        return self._persona_mapping
+
+    @property
+    def agent_list_generator(self):
+        if self._agent_list_generator is None:
+            self._agent_list_generator = self._create_agent_list_generator()
+        return self._agent_list_generator
+
+    @property
+    def results(self):
+        if self._results is None:
+            self._results = self._create_results()
+        return self._results
+
+    def _create_survey(self):
+        survey_pipline_stages = [
+            StageQuestions,
+            StageLabelQuestions,
+            StageGenerateSurvey,
+        ]
+        survey_pipeline = gen_pipeline(survey_pipline_stages)
+        return survey_pipeline.process(
+            data=survey_pipeline.input(
+                overall_question=self.overall_question, population=self.population
+            )
+        ).survey
+
+    def _create_persona_mapping(self):
+        persona_pipeline_stages = [
+            StageQuestions,
+            StagePersona,
+            StagePersonaDimensions,
+            StagePersonaDimensionValues,
+            StagePersonaDimensionValueRanges,
+        ]
+
+        persona_pipeline = gen_pipeline(persona_pipeline_stages)
+        sample_agent_results = persona_pipeline.process(
+            persona_pipeline.input(
+                overall_question=overall_question, population=self.population
+            )
+        )
+        return sample_agent_results
+
+    def _create_agent_list_generator(self):
+        return agent_generator(
+            persona=self.persona_mapping.persona,
+            dimension_dict=self.persona_mapping.mapping,
+        )
+
+    def agent_list(self, num_agents):
+        return create_agents(
+            agent_generator=self.agent_list_generator,
+            survey=self.survey,
+            num_agents=num_agents,
+        )
+
+    def _create_results(self, num_agents=None):
+        if num_agents is None:
+            num_agents = self.default_num_agents
+        agent_list = self.agent_list(num_agents)
+        return self.survey.by(agent_list).by(self.model).run()
+
+
+if __name__ == "__main__":
+    overall_question = "Should online platforms be regulated with respect to selling electric scooters?"
+    auto_study = AutoStudy(overall_question, population="US Adults")
+
+    results = auto_study.results