edsl 0.1.15__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/agents/InvigilatorBase.py

@@ -0,0 +1,257 @@
+from abc import ABC, abstractmethod
+import asyncio
+from typing import Coroutine, Dict, Any, Optional, TYPE_CHECKING
+
+from edsl.utilities.decorators import jupyter_nb_handler
+from edsl.data_transfer_models import AgentResponseDict
+
+if TYPE_CHECKING:
+    from edsl.prompts.Prompt import Prompt
+    from edsl.data.Cache import Cache
+    from edsl.questions.QuestionBase import QuestionBase
+    from edsl.scenarios.Scenario import Scenario
+    from edsl.surveys.MemoryPlan import MemoryPlan
+    from edsl.language_models.LanguageModel import LanguageModel
+    from edsl.surveys.Survey import Survey
+    from edsl.agents.Agent import Agent
+    from edsl.language_models.key_management.KeyLookup import KeyLookup
+
+from edsl.data_transfer_models import EDSLResultObjectInput
+from edsl.agents.PromptConstructor import PromptConstructor
+from edsl.agents.prompt_helpers import PromptPlan
+
+
+class InvigilatorBase(ABC):
+    """An invigiator (someone who administers an exam) is a class that is responsible for administering a question to an agent.
+
+    >>> InvigilatorBase.example().answer_question()
+    {'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
+
+    >>> InvigilatorBase.example().get_failed_task_result(failure_reason="Failed to get response").comment
+    'Failed to get response'
+
+    This returns an empty prompt because there is no memory the agent needs to have at q0.
+    """
+
+    def __init__(
+        self,
+        agent: "Agent",
+        question: "QuestionBase",
+        scenario: "Scenario",
+        model: "LanguageModel",
+        memory_plan: "MemoryPlan",
+        current_answers: dict,
+        survey: Optional["Survey"],
+        cache: Optional["Cache"] = None,
+        iteration: Optional[int] = 1,
+        additional_prompt_data: Optional[dict] = None,
+        raise_validation_errors: Optional[bool] = True,
+        prompt_plan: Optional["PromptPlan"] = None,
+        key_lookup: Optional["KeyLookup"] = None,
+    ):
+        """Initialize a new Invigilator."""
+        self.agent = agent
+        self.question = question
+        self.scenario = scenario
+        self.model = model
+        self.memory_plan = memory_plan
+        self.current_answers = current_answers or {}
+        self.iteration = iteration
+        self.additional_prompt_data = additional_prompt_data
+        self.cache = cache
+        self.survey = survey
+        self.raise_validation_errors = raise_validation_errors
+        self.key_lookup = key_lookup
+
+        if prompt_plan is None:
+            self.prompt_plan = PromptPlan()
+        else:
+            self.prompt_plan = prompt_plan
+
+        # placeholder to store the raw model response
+        self.raw_model_response = None
+
+    @property
+    def prompt_constructor(self) -> PromptConstructor:
+        """Return the prompt constructor."""
+        return PromptConstructor(self, prompt_plan=self.prompt_plan)
+
+    def to_dict(self, include_cache=False) -> Dict[str, Any]:
+        attributes = [
+            "agent",
+            "question",
+            "scenario",
+            "model",
+            "memory_plan",
+            "current_answers",
+            "iteration",
+            "additional_prompt_data",
+            "survey",
+        ]
+        if include_cache:
+            attributes.append("cache")
+
+        def serialize_attribute(attr):
+            value = getattr(self, attr)
+            if value is None:
+                return None
+            if hasattr(value, "to_dict"):
+                return value.to_dict()
+            if isinstance(value, (int, float, str, bool, dict, list)):
+                return value
+            return str(value)
+
+        return {attr: serialize_attribute(attr) for attr in attributes}
+
+    @classmethod
+    def from_dict(cls, data) -> "InvigilatorBase":
+        from edsl.agents.Agent import Agent
+        from edsl.questions import QuestionBase
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.surveys.MemoryPlan import MemoryPlan
+        from edsl.language_models.LanguageModel import LanguageModel
+        from edsl.surveys.Survey import Survey
+        from edsl.data.Cache import Cache
+
+        attributes_to_classes = {
+            "agent": Agent,
+            "question": QuestionBase,
+            "scenario": Scenario,
+            "model": LanguageModel,
+            "memory_plan": MemoryPlan,
+            "survey": Survey,
+            "cache": Cache,
+        }
+        d = {}
+        for attr, cls_ in attributes_to_classes.items():
+            if attr in data and data[attr] is not None:
+                if attr not in data:
+                    d[attr] = {}
+                else:
+                    d[attr] = cls_.from_dict(data[attr])
+
+        d["current_answers"] = data["current_answers"]
+        d["iteration"] = data["iteration"]
+        d["additional_prompt_data"] = data["additional_prompt_data"]
+
+        d = cls(**d)
+
+    def __repr__(self) -> str:
+        """Return a string representation of the Invigilator.
+
+        >>> InvigilatorBase.example().__repr__()
+        'InvigilatorExample(...)'
+
+        """
+        return f"{self.__class__.__name__}(agent={repr(self.agent)}, question={repr(self.question)}, scneario={repr(self.scenario)}, model={repr(self.model)}, memory_plan={repr(self.memory_plan)}, current_answers={repr(self.current_answers)}, iteration{repr(self.iteration)}, additional_prompt_data={repr(self.additional_prompt_data)}, cache={repr(self.cache)})"
+
+    def get_failed_task_result(self, failure_reason: str) -> EDSLResultObjectInput:
+        """Return an AgentResponseDict used in case the question-asking fails.
+
+        Possible reasons include:
+        - Legimately skipped because of skip logic
+        - Failed to get response from the model
+
+        """
+        data = {
+            "answer": None,
+            "generated_tokens": None,
+            "comment": failure_reason,
+            "question_name": self.question.question_name,
+            "prompts": self.get_prompts(),
+            "cached_response": None,
+            "raw_model_response": None,
+            "cache_used": None,
+            "cache_key": None,
+        }
+        return EDSLResultObjectInput(**data)
+
+    def get_prompts(self) -> Dict[str, "Prompt"]:
+        """Return the prompt used."""
+        from edsl.prompts.Prompt import Prompt
+
+        return {
+            "user_prompt": Prompt("NA"),
+            "system_prompt": Prompt("NA"),
+        }
+
+    @abstractmethod
+    async def async_answer_question(self):
+        """Asnwer a question."""
+        pass
+
+    @jupyter_nb_handler
+    def answer_question(self) -> Coroutine:
+        """Return a function that gets the answers to the question."""
+
+        async def main():
+            """Return the answer to the question."""
+            results = await asyncio.gather(self.async_answer_question())
+            return results[0]  # Since there's only one task, return its result
+
+        return main()
+
+    @classmethod
+    def example(
+        cls, throw_an_exception=False, question=None, scenario=None, survey=None
+    ) -> "InvigilatorBase":
+        """Return an example invigilator.
+
+        >>> InvigilatorBase.example()
+        InvigilatorExample(...)
+
+        >>> InvigilatorBase.example().answer_question()
+        {'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
+
+        >>> InvigilatorBase.example(throw_an_exception=True).answer_question()
+        Traceback (most recent call last):
+        ...
+        Exception: This is a test error
+        """
+        from edsl.agents.Agent import Agent
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.surveys.MemoryPlan import MemoryPlan
+        from edsl.language_models.model import Model
+        from edsl.surveys.Survey import Survey
+
+        model = Model("test", canned_response="SPAM!")
+
+        if throw_an_exception:
+            model.throw_exception = True
+        agent = Agent.example()
+
+        if not survey:
+            survey = Survey.example()
+
+        if question not in survey.questions and question is not None:
+            survey.add_question(question)
+
+        question = question or survey.questions[0]
+        scenario = scenario or Scenario.example()
+        memory_plan = MemoryPlan(survey=survey)
+        current_answers = None
+
+        class InvigilatorExample(cls):
+            """An example invigilator."""
+
+            async def async_answer_question(self):
+                """Answer a question."""
+                return await self.model.async_execute_model_call(
+                    user_prompt="Hello", system_prompt="Hi"
+                )
+
+        return InvigilatorExample(
+            agent=agent,
+            question=question,
+            scenario=scenario,
+            survey=survey,
+            model=model,
+            memory_plan=memory_plan,
+            current_answers=current_answers,
+        )
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/agents/PromptConstructor.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+from typing import Dict, Any, Optional, Set, Union, TYPE_CHECKING
+from functools import cached_property
+
+from edsl.prompts.Prompt import Prompt
+
+from dataclasses import dataclass
+
+from .prompt_helpers import PromptPlan
+from .QuestionTemplateReplacementsBuilder import (
+    QuestionTemplateReplacementsBuilder,
+)
+from .question_option_processor import QuestionOptionProcessor
+
+if TYPE_CHECKING:
+    from edsl.agents.InvigilatorBase import InvigilatorBase
+    from edsl.questions.QuestionBase import QuestionBase
+    from edsl.agents.Agent import Agent
+    from edsl.surveys.Survey import Survey
+    from edsl.language_models.LanguageModel import LanguageModel
+    from edsl.surveys.MemoryPlan import MemoryPlan
+    from edsl.questions.QuestionBase import QuestionBase
+    from edsl.scenarios.Scenario import Scenario
+
+
+class BasePlaceholder:
+    """Base class for placeholder values when a question is not yet answered."""
+
+    def __init__(self, placeholder_type: str = "answer"):
+        self.value = "N/A"
+        self.comment = "Will be populated by prior answer"
+        self._type = placeholder_type
+
+    def __getitem__(self, index):
+        return ""
+
+    def __str__(self):
+        return f"<<{self.__class__.__name__}:{self._type}>>"
+
+    def __repr__(self):
+        return self.__str__()
+
+
+class PlaceholderAnswer(BasePlaceholder):
+    def __init__(self):
+        super().__init__("answer")
+
+
+class PlaceholderComment(BasePlaceholder):
+    def __init__(self):
+        super().__init__("comment")
+
+
+class PlaceholderGeneratedTokens(BasePlaceholder):
+    def __init__(self):
+        super().__init__("generated_tokens")
+
+
+class PromptConstructor:
+    """
+    This class constructs the prompts for the language model.
+
+    The pieces of a prompt are:
+    - The agent instructions - "You are answering questions as if you were a human. Do not break character."
+    - The persona prompt - "You are an agent with the following persona: {'age': 22, 'hair': 'brown', 'height': 5.5}"
+    - The question instructions - "You are being asked the following question: Do you like school? The options are 0: yes 1: no Return a valid JSON formatted like this, selecting only the number of the option: {"answer": <put answer code here>, "comment": "<put explanation here>"} Only 1 option may be selected."
+    - The memory prompt - "Before the question you are now answering, you already answered the following question(s): Question: Do you like school? Answer: Prior answer"
+    """
+
+    def __init__(
+        self, invigilator: "InvigilatorBase", prompt_plan: Optional["PromptPlan"] = None
+    ):
+        self.invigilator = invigilator
+        self.prompt_plan = prompt_plan or PromptPlan()
+
+        self.agent = invigilator.agent
+        self.question = invigilator.question
+        self.scenario = invigilator.scenario
+        self.survey = invigilator.survey
+        self.model = invigilator.model
+        self.current_answers = invigilator.current_answers
+        self.memory_plan = invigilator.memory_plan
+
+    def get_question_options(self, question_data):
+        """Get the question options."""
+        return QuestionOptionProcessor(self).get_question_options(question_data)
+
+    @cached_property
+    def agent_instructions_prompt(self) -> Prompt:
+        """
+        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
+        >>> i = InvigilatorBase.example()
+        >>> i.prompt_constructor.agent_instructions_prompt
+        Prompt(text=\"""You are answering questions as if you were a human. Do not break character.\""")
+        """
+        from edsl.agents.Agent import Agent
+
+        if self.agent == Agent():  # if agent is empty, then return an empty prompt
+            return Prompt(text="")
+
+        return Prompt(text=self.agent.instruction)
+
+    @cached_property
+    def agent_persona_prompt(self) -> Prompt:
+        """
+        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
+        >>> i = InvigilatorBase.example()
+        >>> i.prompt_constructor.agent_persona_prompt
+        Prompt(text=\"""Your traits: {'age': 22, 'hair': 'brown', 'height': 5.5}\""")
+        """
+        from edsl.agents.Agent import Agent
+
+        if self.agent == Agent():  # if agent is empty, then return an empty prompt
+            return Prompt(text="")
+
+        return self.agent.prompt()
+
+    def prior_answers_dict(self) -> dict[str, "QuestionBase"]:
+        """This is a dictionary of prior answers, if they exist."""
+        return self._add_answers(
+            self.survey.question_names_to_questions(), self.current_answers
+        )
+
+    @staticmethod
+    def _extract_quetion_and_entry_type(key_entry) -> tuple[str, str]:
+        """
+        Extracts the question name and type for the current answer dictionary key entry.
+
+        >>> PromptConstructor._extract_quetion_and_entry_type("q0")
+        ('q0', 'answer')
+        >>> PromptConstructor._extract_quetion_and_entry_type("q0_comment")
+        ('q0', 'comment')
+        >>> PromptConstructor._extract_quetion_and_entry_type("q0_alternate_generated_tokens")
+        ('q0_alternate', 'generated_tokens')
+        >>> PromptConstructor._extract_quetion_and_entry_type("q0_alt_comment")
+        ('q0_alt', 'comment')
+        """
+        split_list = key_entry.rsplit("_", maxsplit=1)
+        if len(split_list) == 1:
+            question_name = split_list[0]
+            entry_type = "answer"
+        else:
+            if split_list[1] == "comment":
+                question_name = split_list[0]
+                entry_type = "comment"
+            elif split_list[1] == "tokens":  # it's actually 'generated_tokens'
+                question_name = key_entry.replace("_generated_tokens", "")
+                entry_type = "generated_tokens"
+            else:
+                question_name = key_entry
+                entry_type = "answer"
+        return question_name, entry_type
+
+    @staticmethod
+    def _augmented_answers_dict(current_answers: dict) -> dict:
+        """
+        >>> PromptConstructor._augmented_answers_dict({"q0": "LOVE IT!", "q0_comment": "I love school!"})
+        {'q0': {'answer': 'LOVE IT!', 'comment': 'I love school!'}}
+        """
+        from collections import defaultdict
+
+        d = defaultdict(dict)
+        for key, value in current_answers.items():
+            question_name, entry_type = (
+                PromptConstructor._extract_quetion_and_entry_type(key)
+            )
+            d[question_name][entry_type] = value
+        return dict(d)
+
+    @staticmethod
+    def _add_answers(
+        answer_dict: dict, current_answers: dict
+    ) -> dict[str, "QuestionBase"]:
+        """
+        >>> from edsl import QuestionFreeText
+        >>> d = {"q0": QuestionFreeText(question_text="Do you like school?", question_name = "q0")}
+        >>> current_answers = {"q0": "LOVE IT!"}
+        >>> PromptConstructor._add_answers(d, current_answers)['q0'].answer
+        'LOVE IT!'
+        """
+        augmented_answers = PromptConstructor._augmented_answers_dict(current_answers)
+
+        for question in answer_dict:
+            if question in augmented_answers:
+                for entry_type, value in augmented_answers[question].items():
+                    setattr(answer_dict[question], entry_type, value)
+            else:
+                answer_dict[question].answer = PlaceholderAnswer()
+                answer_dict[question].comment = PlaceholderComment()
+                answer_dict[question].generated_tokens = PlaceholderGeneratedTokens()
+        return answer_dict
+
+    @cached_property
+    def question_file_keys(self) -> list:
+        """Extracts the file keys from the question text.
+        It checks if the variables in the question text are in the scenario file keys.
+        """
+        return QuestionTemplateReplacementsBuilder(self).question_file_keys()
+
+    @cached_property
+    def question_instructions_prompt(self) -> Prompt:
+        """
+        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
+        >>> i = InvigilatorBase.example()
+        >>> i.prompt_constructor.question_instructions_prompt
+        Prompt(text=\"""...
+        ...
+        """
+        return self.build_question_instructions_prompt()
+
+    def build_question_instructions_prompt(self) -> Prompt:
+        """Buils the question instructions prompt."""
+        from edsl.agents.QuestionInstructionPromptBuilder import (
+            QuestionInstructionPromptBuilder,
+        )
+
+        return QuestionInstructionPromptBuilder(self).build()
+
+    @cached_property
+    def prior_question_memory_prompt(self) -> Prompt:
+        memory_prompt = Prompt(text="")
+        if self.memory_plan is not None:
+            memory_prompt += self.create_memory_prompt(
+                self.question.question_name
+            ).render(self.scenario | self.prior_answers_dict())
+        return memory_prompt
+
+    def create_memory_prompt(self, question_name: str) -> Prompt:
+        """Create a memory for the agent.
+
+        The returns a memory prompt for the agent.
+
+        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
+        >>> i = InvigilatorBase.example()
+        >>> i.current_answers = {"q0": "Prior answer"}
+        >>> i.memory_plan.add_single_memory("q1", "q0")
+        >>> p = i.prompt_constructor.create_memory_prompt("q1")
+        >>> p.text.strip().replace("\\n", " ").replace("\\t", " ")
+        'Before the question you are now answering, you already answered the following question(s):          Question: Do you like school?  Answer: Prior answer'
+        """
+        return self.memory_plan.get_memory_prompt_fragment(
+            question_name, self.current_answers
+        )
+
+    def get_prompts(self) -> Dict[str, Prompt]:
+        """Get both prompts for the LLM call.
+
+        >>> from edsl import QuestionFreeText
+        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
+        >>> q = QuestionFreeText(question_text="How are you today?", question_name="q_new")
+        >>> i = InvigilatorBase.example(question = q)
+        >>> i.get_prompts()
+        {'user_prompt': ..., 'system_prompt': ...}
+        """
+        prompts = self.prompt_plan.get_prompts(
+            agent_instructions=self.agent_instructions_prompt,
+            agent_persona=self.agent_persona_prompt,
+            question_instructions=Prompt(self.question_instructions_prompt),
+            prior_question_memory=self.prior_question_memory_prompt,
+        )
+        if self.question_file_keys:
+            files_list = []
+            for key in self.question_file_keys:
+                files_list.append(self.scenario[key])
+            prompts["files_list"] = files_list
+        return prompts
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/agents/QuestionInstructionPromptBuilder.py

@@ -0,0 +1,128 @@
+from typing import Dict, List, Set
+from warnings import warn
+from edsl.prompts.Prompt import Prompt
+
+from edsl.agents.QuestionTemplateReplacementsBuilder import (
+    QuestionTemplateReplacementsBuilder as QTRB,
+)
+
+
+class QuestionInstructionPromptBuilder:
+    """Handles the construction and rendering of question instructions."""
+
+    def __init__(self, prompt_constructor: "PromptConstructor"):
+        self.prompt_constructor = prompt_constructor
+
+        self.model = self.prompt_constructor.model
+        self.survey = self.prompt_constructor.survey
+        self.question = self.prompt_constructor.question
+
+    def build(self) -> Prompt:
+        """Builds the complete question instructions prompt with all necessary components.
+
+        Returns:
+            Prompt: The fully rendered question instructions
+        """
+        base_prompt = self._create_base_prompt()
+        enriched_prompt = self._enrich_with_question_options(base_prompt)
+        rendered_prompt = self._render_prompt(enriched_prompt)
+        self._validate_template_variables(rendered_prompt)
+
+        return self._append_survey_instructions(rendered_prompt)
+
+    def _create_base_prompt(self) -> Dict:
+        """Creates the initial prompt with basic question data.
+
+        Returns:
+            Dict: Base question data
+        """
+        return {
+            "prompt": Prompt(self.question.get_instructions(model=self.model.model)),
+            "data": self.question.data.copy(),
+        }
+
+    def _enrich_with_question_options(self, prompt_data: Dict) -> Dict:
+        """Enriches the prompt data with question options if they exist.
+
+        Args:
+            prompt_data: Dictionary containing prompt and question data
+
+        Returns:
+            Dict: Enriched prompt data
+        """
+        if "question_options" in prompt_data["data"]:
+            from edsl.agents.question_option_processor import QuestionOptionProcessor
+
+            question_options = QuestionOptionProcessor(
+                self.prompt_constructor
+            ).get_question_options(question_data=prompt_data["data"])
+
+            prompt_data["data"]["question_options"] = question_options
+        return prompt_data
+
+    def _render_prompt(self, prompt_data: Dict) -> Prompt:
+        """Renders the prompt using the replacement dictionary.
+
+        Args:
+            prompt_data: Dictionary containing prompt and question data
+
+        Returns:
+            Prompt: Rendered instructions
+        """
+
+        replacement_dict = QTRB(self.prompt_constructor).build_replacement_dict(
+            prompt_data["data"]
+        )
+        return prompt_data["prompt"].render(replacement_dict)
+
+    def _validate_template_variables(self, rendered_prompt: Prompt) -> None:
+        """Validates that all template variables have been properly replaced.
+
+        Args:
+            rendered_prompt: The rendered prompt to validate
+
+        Warns:
+            If any template variables remain undefined
+        """
+        undefined_vars = rendered_prompt.undefined_template_variables({})
+
+        # Check for question names in undefined variables
+        self._check_question_names_in_undefined_vars(undefined_vars)
+
+        # Warn about any remaining undefined variables
+        if undefined_vars:
+            warn(f"Question instructions still has variables: {undefined_vars}.")
+
+    def _check_question_names_in_undefined_vars(self, undefined_vars: Set[str]) -> None:
+        """Checks if any undefined variables match question names in the survey.
+
+        Args:
+            undefined_vars: Set of undefined template variables
+        """
+        for question_name in self.survey.question_names:
+            if question_name in undefined_vars:
+                print(
+                    f"Question name found in undefined_template_variables: {question_name}"
+                )
+
+    def _append_survey_instructions(self, rendered_prompt: Prompt) -> Prompt:
+        """Appends any relevant survey instructions to the rendered prompt.
+
+        Args:
+            rendered_prompt: The rendered prompt to append instructions to
+
+        Returns:
+            Prompt: Final prompt with survey instructions
+        """
+        relevant_instructions = self.survey._relevant_instructions(
+            self.question.question_name
+        )
+
+        if not relevant_instructions:
+            return rendered_prompt
+
+        preamble = Prompt(text="")
+        for instruction in relevant_instructions:
+            preamble += instruction.text
+
+        return preamble + rendered_prompt