edsl 0.1.46__py3-none-any.whl → 0.1.48__py3-none-any.whl

edsl/agents/InvigilatorBase.py

@@ -1,260 +0,0 @@
-from abc import ABC, abstractmethod
-import asyncio
-from typing import Coroutine, Dict, Any, Optional, TYPE_CHECKING
-
-from edsl.utilities.decorators import jupyter_nb_handler
-from edsl.data_transfer_models import AgentResponseDict
-
-if TYPE_CHECKING:
-    from edsl.prompts.Prompt import Prompt
-    from edsl.data.Cache import Cache
-    from edsl.questions.QuestionBase import QuestionBase
-    from edsl.scenarios.Scenario import Scenario
-    from edsl.surveys.MemoryPlan import MemoryPlan
-    from edsl.language_models.LanguageModel import LanguageModel
-    from edsl.surveys.Survey import Survey
-    from edsl.agents.Agent import Agent
-    from edsl.language_models.key_management.KeyLookup import KeyLookup
-
-from edsl.data_transfer_models import EDSLResultObjectInput
-from edsl.agents.PromptConstructor import PromptConstructor
-from edsl.agents.prompt_helpers import PromptPlan
-
-
-class InvigilatorBase(ABC):
-    """An invigiator (someone who administers an exam) is a class that is responsible for administering a question to an agent.
-
-    >>> InvigilatorBase.example().answer_question()
-    {'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
-
-    >>> InvigilatorBase.example().get_failed_task_result(failure_reason="Failed to get response").comment
-    'Failed to get response'
-
-    This returns an empty prompt because there is no memory the agent needs to have at q0.
-    """
-
-    def __init__(
-        self,
-        agent: "Agent",
-        question: "QuestionBase",
-        scenario: "Scenario",
-        model: "LanguageModel",
-        memory_plan: "MemoryPlan",
-        current_answers: dict,
-        survey: Optional["Survey"],
-        cache: Optional["Cache"] = None,
-        iteration: Optional[int] = 1,
-        additional_prompt_data: Optional[dict] = None,
-        raise_validation_errors: Optional[bool] = True,
-        prompt_plan: Optional["PromptPlan"] = None,
-        key_lookup: Optional["KeyLookup"] = None,
-    ):
-        """Initialize a new Invigilator."""
-        self.agent = agent
-        self.question = question
-        self.scenario = scenario
-        self.model = model
-        self.memory_plan = memory_plan
-        self.current_answers = current_answers or {}
-        self.iteration = iteration
-        self.additional_prompt_data = additional_prompt_data
-        self.cache = cache
-        self.survey = survey
-        self.raise_validation_errors = raise_validation_errors
-        self.key_lookup = key_lookup
-
-        if prompt_plan is None:
-            self.prompt_plan = PromptPlan()
-        else:
-            self.prompt_plan = prompt_plan
-
-        # placeholder to store the raw model response
-        self.raw_model_response = None
-
-    @property
-    def prompt_constructor(self) -> PromptConstructor:
-        """Return the prompt constructor."""
-        return PromptConstructor.from_invigilator(self, prompt_plan=self.prompt_plan)
-
-    def to_dict(self, include_cache=False) -> Dict[str, Any]:
-        attributes = [
-            "agent",
-            "question",
-            "scenario",
-            "model",
-            "memory_plan",
-            "current_answers",
-            "iteration",
-            "additional_prompt_data",
-            "survey",
-            "raw_model_response",
-        ]
-        if include_cache:
-            attributes.append("cache")
-
-        def serialize_attribute(attr):
-            value = getattr(self, attr)
-            if value is None:
-                return None
-            if hasattr(value, "to_dict"):
-                return value.to_dict()
-            if isinstance(value, (int, float, str, bool, dict, list)):
-                return value
-            return str(value)
-
-        return {attr: serialize_attribute(attr) for attr in attributes}
-
-    @classmethod
-    def from_dict(cls, data) -> "InvigilatorBase":
-        from edsl.agents.Agent import Agent
-        from edsl.questions import QuestionBase
-        from edsl.scenarios.Scenario import Scenario
-        from edsl.surveys.MemoryPlan import MemoryPlan
-        from edsl.language_models.LanguageModel import LanguageModel
-        from edsl.surveys.Survey import Survey
-        from edsl.data.Cache import Cache
-
-        attributes_to_classes = {
-            "agent": Agent,
-            "question": QuestionBase,
-            "scenario": Scenario,
-            "model": LanguageModel,
-            "memory_plan": MemoryPlan,
-            "survey": Survey,
-            "cache": Cache,
-        }
-        d = {}
-        for attr, cls_ in attributes_to_classes.items():
-            if attr in data and data[attr] is not None:
-                if attr not in data:
-                    d[attr] = {}
-                else:
-                    d[attr] = cls_.from_dict(data[attr])
-
-        d["current_answers"] = data["current_answers"]
-        d["iteration"] = data["iteration"]
-        d["additional_prompt_data"] = data["additional_prompt_data"]
-
-        d = cls(**d)
-        d.raw_model_response = data.get("raw_model_response")
-        return d
-
-    def __repr__(self) -> str:
-        """Return a string representation of the Invigilator.
-
-        >>> InvigilatorBase.example().__repr__()
-        'InvigilatorExample(...)'
-
-        """
-        return f"{self.__class__.__name__}(agent={repr(self.agent)}, question={repr(self.question)}, scenario={repr(self.scenario)}, model={repr(self.model)}, memory_plan={repr(self.memory_plan)}, current_answers={repr(self.current_answers)}, iteration={repr(self.iteration)}, additional_prompt_data={repr(self.additional_prompt_data)}, cache={repr(self.cache)})"
-
-    def get_failed_task_result(self, failure_reason: str) -> EDSLResultObjectInput:
-        """Return an AgentResponseDict used in case the question-asking fails.
-
-        Possible reasons include:
-        - Legimately skipped because of skip logic
-        - Failed to get response from the model
-
-        """
-        data = {
-            "answer": None,
-            "generated_tokens": None,
-            "comment": failure_reason,
-            "question_name": self.question.question_name,
-            "prompts": self.get_prompts(),
-            "cached_response": None,
-            "raw_model_response": None,
-            "cache_used": None,
-            "cache_key": None,
-        }
-        return EDSLResultObjectInput(**data)
-
-    def get_prompts(self) -> Dict[str, "Prompt"]:
-        """Return the prompt used."""
-        from edsl.prompts.Prompt import Prompt
-
-        return {
-            "user_prompt": Prompt("NA"),
-            "system_prompt": Prompt("NA"),
-        }
-
-    @abstractmethod
-    async def async_answer_question(self):
-        """Asnwer a question."""
-        pass
-
-    @jupyter_nb_handler
-    def answer_question(self) -> Coroutine:
-        """Return a function that gets the answers to the question."""
-
-        async def main():
-            """Return the answer to the question."""
-            results = await asyncio.gather(self.async_answer_question())
-            return results[0]  # Since there's only one task, return its result
-
-        return main()
-
-    @classmethod
-    def example(
-        cls, throw_an_exception=False, question=None, scenario=None, survey=None
-    ) -> "InvigilatorBase":
-        """Return an example invigilator.
-
-        >>> InvigilatorBase.example()
-        InvigilatorExample(...)
-
-        >>> InvigilatorBase.example().answer_question()
-        {'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
-
-        >>> InvigilatorBase.example(throw_an_exception=True).answer_question()
-        Traceback (most recent call last):
-        ...
-        Exception: This is a test error
-        """
-        from edsl.agents.Agent import Agent
-        from edsl.scenarios.Scenario import Scenario
-        from edsl.surveys.MemoryPlan import MemoryPlan
-        from edsl.language_models.model import Model
-        from edsl.surveys.Survey import Survey
-
-        model = Model("test", canned_response="SPAM!")
-
-        if throw_an_exception:
-            model.throw_exception = True
-        agent = Agent.example()
-
-        if not survey:
-            survey = Survey.example()
-
-        if question not in survey.questions and question is not None:
-            survey.add_question(question)
-
-        question = question or survey.questions[0]
-        scenario = scenario or Scenario.example()
-        memory_plan = MemoryPlan(survey=survey)
-        current_answers = None
-
-        class InvigilatorExample(cls):
-            """An example invigilator."""
-
-            async def async_answer_question(self):
-                """Answer a question."""
-                return await self.model.async_execute_model_call(
-                    user_prompt="Hello", system_prompt="Hi"
-                )
-
-        return InvigilatorExample(
-            agent=agent,
-            question=question,
-            scenario=scenario,
-            survey=survey,
-            model=model,
-            memory_plan=memory_plan,
-            current_answers=current_answers,
-        )
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/agents/PromptConstructor.py

@@ -1,318 +0,0 @@
-from __future__ import annotations
-from typing import Dict, Any, Optional, Set, Union, TYPE_CHECKING, Literal
-from functools import cached_property
-import time
-import logging
-
-from edsl.prompts.Prompt import Prompt
-
-from edsl.agents.prompt_helpers import PromptPlan
-from edsl.agents.QuestionTemplateReplacementsBuilder import (
-    QuestionTemplateReplacementsBuilder,
-)
-from edsl.agents.question_option_processor import QuestionOptionProcessor
-
-if TYPE_CHECKING:
-    from edsl.agents.InvigilatorBase import InvigilatorBase
-    from edsl.questions.QuestionBase import QuestionBase
-    from edsl.agents.Agent import Agent
-    from edsl.surveys.Survey import Survey
-    from edsl.language_models.LanguageModel import LanguageModel
-    from edsl.surveys.MemoryPlan import MemoryPlan
-    from edsl.questions.QuestionBase import QuestionBase
-    from edsl.scenarios.Scenario import Scenario
-
-logger = logging.getLogger(__name__)
-
-class BasePlaceholder:
-    """Base class for placeholder values when a question is not yet answered."""
-
-    def __init__(self, placeholder_type: str = "answer"):
-        self.value = "N/A"
-        self.comment = "Will be populated by prior answer"
-        self._type = placeholder_type
-
-    def __getitem__(self, index):
-        return ""
-
-    def __str__(self):
-        return f"<<{self.__class__.__name__}:{self._type}>>"
-
-    def __repr__(self):
-        return self.__str__()
-
-
-class PlaceholderAnswer(BasePlaceholder):
-    def __init__(self):
-        super().__init__("answer")
-
-
-class PlaceholderComment(BasePlaceholder):
-    def __init__(self):
-        super().__init__("comment")
-
-
-class PlaceholderGeneratedTokens(BasePlaceholder):
-    def __init__(self):
-        super().__init__("generated_tokens")
-
-
-class PromptConstructor:
-    """
-    This class constructs the prompts for the language model.
-
-    The pieces of a prompt are:
-    - The agent instructions - "You are answering questions as if you were a human. Do not break character."
-    - The persona prompt - "You are an agent with the following persona: {'age': 22, 'hair': 'brown', 'height': 5.5}"
-    - The question instructions - "You are being asked the following question: Do you like school? The options are 0: yes 1: no Return a valid JSON formatted like this, selecting only the number of the option: {"answer": <put answer code here>, "comment": "<put explanation here>"} Only 1 option may be selected."
-    - The memory prompt - "Before the question you are now answering, you already answered the following question(s): Question: Do you like school? Answer: Prior answer"
-    """
-    @classmethod
-    def from_invigilator(
-        cls,
-        invigilator: "InvigilatorBase",
-        prompt_plan: Optional["PromptPlan"] = None
-    ) -> "PromptConstructor":
-        return cls(
-            agent=invigilator.agent,
-            question=invigilator.question,
-            scenario=invigilator.scenario,
-            survey=invigilator.survey,
-            model=invigilator.model,
-            current_answers=invigilator.current_answers,
-            memory_plan=invigilator.memory_plan,
-            prompt_plan=prompt_plan
-        )
-
-    def __init__(
-        self,
-        agent: "Agent",
-        question: "QuestionBase",
-        scenario: "Scenario",
-        survey: "Survey",
-        model: "LanguageModel",
-        current_answers: dict,
-        memory_plan: "MemoryPlan",
-        prompt_plan: Optional["PromptPlan"] = None
-    ):
-        self.agent = agent
-        self.question = question
-        self.scenario = scenario
-        self.survey = survey
-        self.model = model
-        self.current_answers = current_answers
-        self.memory_plan = memory_plan
-        self.prompt_plan = prompt_plan or PromptPlan()
-
-    def get_question_options(self, question_data: dict) -> list[str]:
-        """Get the question options."""
-        return (QuestionOptionProcessor
-                .from_prompt_constructor(self)
-                .get_question_options(question_data)
-        )
-
-    @cached_property
-    def agent_instructions_prompt(self) -> Prompt:
-        """
-        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> i = InvigilatorBase.example()
-        >>> i.prompt_constructor.agent_instructions_prompt
-        Prompt(text=\"""You are answering questions as if you were a human. Do not break character.\""")
-        """
-        from edsl.agents.Agent import Agent
-
-        if self.agent == Agent():  # if agent is empty, then return an empty prompt
-            return Prompt(text="")
-
-        return Prompt(text=self.agent.instruction)
-
-    @cached_property
-    def agent_persona_prompt(self) -> Prompt:
-        """
-        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> i = InvigilatorBase.example()
-        >>> i.prompt_constructor.agent_persona_prompt
-        Prompt(text=\"""Your traits: {'age': 22, 'hair': 'brown', 'height': 5.5}\""")
-        """
-        from edsl.agents.Agent import Agent
-
-        if self.agent == Agent():  # if agent is empty, then return an empty prompt
-            return Prompt(text="")
-
-        return self.agent.prompt()
-
-    def prior_answers_dict(self) -> dict[str, "QuestionBase"]:
-        """This is a dictionary of prior answers, if they exist.
-        
-        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> i = InvigilatorBase.example()
-        >>> i.prompt_constructor.prior_answers_dict()
-        {'q0': ..., 'q1': ...}
-        """
-        return self._add_answers(
-            self.survey.question_names_to_questions(), self.current_answers
-        )
-
-    @staticmethod
-    def _extract_question_and_entry_type(key_entry) -> tuple[str, str]:
-        """
-        Extracts the question name and type for the current answer dictionary key entry.
-
-        >>> PromptConstructor._extract_question_and_entry_type("q0")
-        ('q0', 'answer')
-        >>> PromptConstructor._extract_question_and_entry_type("q0_comment")
-        ('q0', 'comment')
-        >>> PromptConstructor._extract_question_and_entry_type("q0_alternate_generated_tokens")
-        ('q0_alternate', 'generated_tokens')
-        >>> PromptConstructor._extract_question_and_entry_type("q0_alt_comment")
-        ('q0_alt', 'comment')
-        """
-        split_list = key_entry.rsplit("_", maxsplit=1)
-        if len(split_list) == 1:
-            question_name = split_list[0]
-            entry_type = "answer"
-        else:
-            if split_list[1] == "comment":
-                question_name = split_list[0]
-                entry_type = "comment"
-            elif split_list[1] == "tokens":  # it's actually 'generated_tokens'
-                question_name = key_entry.replace("_generated_tokens", "")
-                entry_type = "generated_tokens"
-            else:
-                question_name = key_entry
-                entry_type = "answer"
-        return question_name, entry_type
-
-    @staticmethod
-    def _augmented_answers_dict(current_answers: dict) -> dict:
-        """
-        >>> PromptConstructor._augmented_answers_dict({"q0": "LOVE IT!", "q0_comment": "I love school!"})
-        {'q0': {'answer': 'LOVE IT!', 'comment': 'I love school!'}}
-        """
-        from collections import defaultdict
-
-        d = defaultdict(dict)
-        for key, value in current_answers.items():
-            question_name, entry_type = (
-                PromptConstructor._extract_question_and_entry_type(key)
-            )
-            d[question_name][entry_type] = value
-        return dict(d)
-
-    @staticmethod
-    def _add_answers(
-        answer_dict: dict, current_answers: dict
-    ) -> dict[str, "QuestionBase"]:
-        """
-        >>> from edsl import QuestionFreeText
-        >>> d = {"q0": QuestionFreeText(question_text="Do you like school?", question_name = "q0")}
-        >>> current_answers = {"q0": "LOVE IT!"}
-        >>> PromptConstructor._add_answers(d, current_answers)['q0'].answer
-        'LOVE IT!'
-        """
-        augmented_answers = PromptConstructor._augmented_answers_dict(current_answers)
-
-        for question in answer_dict:
-            if question in augmented_answers:
-                for entry_type, value in augmented_answers[question].items():
-                    setattr(answer_dict[question], entry_type, value)
-            else:
-                answer_dict[question].answer = PlaceholderAnswer()
-                answer_dict[question].comment = PlaceholderComment()
-                answer_dict[question].generated_tokens = PlaceholderGeneratedTokens()
-        return answer_dict
-
-    @cached_property
-    def question_file_keys(self) -> list:
-        """Extracts the file keys from the question text.
-        
-        It checks if the variables in the question text are in the scenario file keys.
-        """
-        return QuestionTemplateReplacementsBuilder.from_prompt_constructor(self).question_file_keys()
-
-    @cached_property
-    def question_instructions_prompt(self) -> Prompt:
-        """
-        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> i = InvigilatorBase.example()
-        >>> i.prompt_constructor.question_instructions_prompt
-        Prompt(text=\"""...
-        ...
-        """
-        return self.build_question_instructions_prompt()
-
-    def build_question_instructions_prompt(self) -> Prompt:
-        """Buils the question instructions prompt."""
-        from edsl.agents.QuestionInstructionPromptBuilder import (
-            QuestionInstructionPromptBuilder,
-        )
-
-        return QuestionInstructionPromptBuilder.from_prompt_constructor(self).build()
-
-    @cached_property
-    def prior_question_memory_prompt(self) -> Prompt:
-        memory_prompt = Prompt(text="")
-        if self.memory_plan is not None:
-            memory_prompt += self.create_memory_prompt(
-                self.question.question_name
-            ).render(self.scenario | self.prior_answers_dict())
-        return memory_prompt
-
-    def create_memory_prompt(self, question_name: str) -> Prompt:
-        """Create a memory for the agent.
-
-        The returns a memory prompt for the agent.
-
-        >>> from edsl.agents.InvigilatorBase import InvigilatorBase
-        >>> i = InvigilatorBase.example()
-        >>> i.current_answers = {"q0": "Prior answer"}
-        >>> i.memory_plan.add_single_memory("q1", "q0")
-        >>> p = i.prompt_constructor.create_memory_prompt("q1")
-        >>> p.text.strip().replace("\\n", " ").replace("\\t", " ")
-        'Before the question you are now answering, you already answered the following question(s):          Question: Do you like school?  Answer: Prior answer'
-        """
-        return self.memory_plan.get_memory_prompt_fragment(
-            question_name, self.current_answers
-        )
-
-    def get_prompts(self) -> Dict[str, Any]:
-        """Get the prompts for the question."""
-        start = time.time()
-        
-        # Build all the components
-        agent_instructions = self.agent_instructions_prompt
-        agent_persona = self.agent_persona_prompt
-        question_instructions = self.question_instructions_prompt
-        prior_question_memory = self.prior_question_memory_prompt
-        
-        # Get components dict
-        components = {
-            "agent_instructions": agent_instructions.text,
-            "agent_persona": agent_persona.text,
-            "question_instructions": question_instructions.text,
-            "prior_question_memory": prior_question_memory.text,
-        }        
-        # Get arranged components first
-        arranged = self.prompt_plan.arrange_components(**components)
-        
-        prompts = self.prompt_plan.get_prompts(**components)
-        
-        # Handle file keys if present
-        if hasattr(self, 'question_file_keys') and self.question_file_keys:
-            files_list = []
-            for key in self.question_file_keys:
-                files_list.append(self.scenario[key])
-            prompts["files_list"] = files_list
-        
-        return prompts
-
-
-# def _process_prompt(args):
-#     """Helper function to process a single prompt list with its replacements."""
-#     prompt_list, replacements = args
-#     return prompt_list.reduce()
-
-
-if __name__ == '__main__':
-    import doctest
-    doctest.testmod(optionflags=doctest.ELLIPSIS)