edsl 0.1.33__py3-none-any.whl → 0.1.33.dev1__py3-none-any.whl

edsl/Base.py

@@ -47,27 +47,21 @@ class PersistenceMixin:
         self,
         description: Optional[str] = None,
         visibility: Optional[str] = "unlisted",
-        expected_parrot_url: Optional[str] = None,
     ):
         """Post the object to coop."""
         from edsl.coop import Coop
 
-        c = Coop(url=expected_parrot_url)
+        c = Coop()
         return c.create(self, description, visibility)
 
     @classmethod
-    def pull(
-        cls,
-        uuid: Optional[Union[str, UUID]] = None,
-        url: Optional[str] = None,
-        expected_parrot_url: Optional[str] = None,
-    ):
+    def pull(cls, uuid: Optional[Union[str, UUID]] = None, url: Optional[str] = None):
         """Pull the object from coop."""
         from edsl.coop import Coop
         from edsl.coop.utils import ObjectRegistry
 
         object_type = ObjectRegistry.get_object_type_by_edsl_class(cls)
-        coop = Coop(url=expected_parrot_url)
+        coop = Coop()
         return coop.get(uuid, url, object_type)
 
     @classmethod

edsl/__init__.py

@@ -8,10 +8,9 @@ from edsl.__version__ import __version__
 from edsl.config import Config, CONFIG
 from edsl.agents.Agent import Agent
 from edsl.agents.AgentList import AgentList
-
 from edsl.questions import QuestionBase
-from edsl.questions.question_registry import Question
 from edsl.questions import QuestionMultipleChoice
+from edsl.questions import QuestionBudget
 from edsl.questions import QuestionCheckBox
 from edsl.questions import QuestionExtract
 from edsl.questions import QuestionFreeText
@@ -20,11 +19,10 @@ from edsl.questions import QuestionLikertFive
 from edsl.questions import QuestionList
 from edsl.questions import QuestionLinearScale
 from edsl.questions import QuestionNumerical
-from edsl.questions import QuestionYesNo
-from edsl.questions import QuestionBudget
 from edsl.questions import QuestionRank
 from edsl.questions import QuestionTopK
-
+from edsl.questions import QuestionYesNo
+from edsl.questions.question_registry import Question
 from edsl.scenarios import Scenario
 from edsl.scenarios import ScenarioList
 
@@ -42,6 +40,3 @@ from edsl.notebooks.Notebook import Notebook
 from edsl.study.Study import Study
 from edsl.conjure.Conjure import Conjure
 from edsl.coop.coop import Coop
-
-from edsl.surveys.instructions.Instruction import Instruction
-from edsl.surveys.instructions.ChangeInstruction import ChangeInstruction

edsl/__version__.py

@@ -1 +1 @@
-__version__ = "0.1.33"
+__version__ = "0.1.33.dev1"

edsl/agents/Agent.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import copy
 import inspect
 import types
-from typing import Callable, Optional, Union, Any
+from typing import Callable, Optional, Union
 from uuid import uuid4
 from edsl.Base import Base
 
@@ -228,12 +228,7 @@ class Agent(Base):
         if hasattr(self, "answer_question_directly"):
             delattr(self, "answer_question_directly")
 
-    def add_direct_question_answering_method(
-        self,
-        method: Callable,
-        validate_response: bool = False,
-        translate_response: bool = False,
-    ) -> None:
+    def add_direct_question_answering_method(self, method: Callable) -> None:
         """Add a method to the agent that can answer a particular question type.
 
         :param method: A method that can answer a question directly.
@@ -254,9 +249,6 @@ class Agent(Base):
             )
             # print("Warning: overwriting existing answer_question_directly method")
 
-        self.validate_response = validate_response
-        self.translate_response = translate_response
-
         signature = inspect.signature(method)
         for argument in ["question", "scenario", "self"]:
             if argument not in signature.parameters:
@@ -280,7 +272,6 @@ class Agent(Base):
         current_answers: Optional[dict] = None,
         iteration: int = 1,
         sidecar_model=None,
-        raise_validation_errors: bool = True,
     ) -> "InvigilatorBase":
         """Create an Invigilator.
 
@@ -312,12 +303,7 @@ class Agent(Base):
             iteration=iteration,
             cache=cache,
             sidecar_model=sidecar_model,
-            raise_validation_errors=raise_validation_errors,
         )
-        if hasattr(self, "validate_response"):
-            invigilator.validate_response = self.validate_response
-        if hasattr(self, "translate_response"):
-            invigilator.translate_response = self.translate_response
         return invigilator
 
     async def async_answer_question(
@@ -348,8 +334,8 @@ class Agent(Base):
         >>> a.add_direct_question_answering_method(lambda self, question, scenario: "I am a direct answer.")
         >>> from edsl import QuestionFreeText
         >>> q = QuestionFreeText.example()
-        >>> a.answer_question(question = q, cache = False).answer
-        'I am a direct answer.'
+        >>> a.answer_question(question = q, cache = False)
+        {'answer': 'I am a direct answer.', 'comment': 'This is a real survey response from a human.', ...}
 
         This is a function where an agent returns an answer to a particular question.
         However, there are several different ways an agent can answer a question, so the
@@ -383,7 +369,6 @@ class Agent(Base):
         current_answers: Optional[dict] = None,
         iteration: int = 0,
         sidecar_model=None,
-        raise_validation_errors: bool = True,
     ) -> "InvigilatorBase":
         """Create an Invigilator."""
         from edsl import Model
@@ -393,6 +378,7 @@ class Agent(Base):
         scenario = scenario or Scenario()
 
         from edsl.agents.Invigilator import (
+            InvigilatorDebug,
             InvigilatorHuman,
             InvigilatorFunctional,
             InvigilatorAI,
@@ -405,9 +391,8 @@ class Agent(Base):
             cache = Cache()
 
         if debug:
-            raise NotImplementedError("Debug mode is not yet implemented.")
             # use the question's _simulate_answer method
-            # invigilator_class = InvigilatorDebug
+            invigilator_class = InvigilatorDebug
         elif hasattr(question, "answer_question_directly"):
             # It's a functional question and the answer only depends on the agent's traits & the scenario
             invigilator_class = InvigilatorFunctional
@@ -437,7 +422,6 @@ class Agent(Base):
             iteration=iteration,
             cache=cache,
             sidecar_model=sidecar_model,
-            raise_validation_errors=raise_validation_errors,
         )
         return invigilator
 
@@ -513,8 +497,8 @@ class Agent(Base):
         if name == "has_dynamic_traits_function":
             return self.has_dynamic_traits_function
 
-        if name in self._traits:
-            return self._traits[name]
+        if name in self.traits:
+            return self.traits[name]
         raise AttributeError(
             f"'{type(self).__name__}' object has no attribute '{name}'"
         )
@@ -656,22 +640,6 @@ class Agent(Base):
         column_names = ["Attribute", "Value"]
         return table_data, column_names
 
-    def add_trait(self, trait_name_or_dict: str, value: Optional[Any] = None) -> Agent:
-        """Adds a trait to an agent and returns that agent"""
-        if isinstance(trait_name_or_dict, dict) and value is None:
-            self.traits.update(trait_name_or_dict)
-            return self
-
-        if isinstance(trait_name_or_dict, dict) and value:
-            raise ValueError(f"You passed a dict: {trait_name_or_dict}")
-
-        if isinstance(trait_name_or_dict, str):
-            trait = trait_name_or_dict
-            self.traits[trait] = value
-            return self
-
-        raise Exception("Something is not right with adding")
-
     def remove_trait(self, trait: str) -> Agent:
         """Remove a trait from the agent.
 

edsl/agents/AgentList.py

@@ -21,12 +21,6 @@ from simpleeval import EvalWithCompoundTypes
 from edsl.Base import Base
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
-from collections.abc import Iterable
-
-
-def is_iterable(obj):
-    return isinstance(obj, Iterable)
-
 
 class AgentList(UserList, Base):
     """A list of Agents."""
@@ -117,13 +111,6 @@ class AgentList(UserList, Base):
 
         return AgentList(new_data)
 
-    @property
-    def all_traits(self):
-        d = {}
-        for agent in self:
-            d.update(agent.traits)
-        return list(d.keys())
-
     @classmethod
     def from_csv(cls, file_path: str):
         """Load AgentList from a CSV file.
@@ -172,36 +159,6 @@ class AgentList(UserList, Base):
             _ = agent.remove_trait(trait)
         return self
 
-    def add_trait(self, trait, values):
-        """Adds a new trait to every agent, with values taken from values.
-
-        :param trait: The name of the trait.
-        :param values: The valeues(s) of the trait. If a single value is passed, it is used for all agents.
-
-        >>> al = AgentList.example()
-        >>> al.add_trait('new_trait', 1)
-        AgentList([Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5, 'new_trait': 1}), Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5, 'new_trait': 1})])
-        >>> al.select('new_trait').to_scenario_list().to_list()
-        [1, 1]
-        >>> al.add_trait('new_trait', [1, 2, 3])
-        Traceback (most recent call last):
-        ...
-        ValueError: The passed values have to be the same length as the agent list.
-        """
-        if not is_iterable(values):
-            value = values
-            for agent in self.data:
-                agent.add_trait(trait, value)
-            return self
-
-        if len(values) != len(self):
-            raise ValueError(
-                "The passed values have to be the same length as the agent list."
-            )
-        for agent, value in zip(self.data, values):
-            agent.add_trait(trait, value)
-        return self
-
     @staticmethod
     def get_codebook(file_path: str):
         """Return the codebook for a CSV file.

edsl/agents/Invigilator.py

@@ -1,169 +1,252 @@
 """Module for creating Invigilators, which are objects to administer a question to an Agent."""
 
+import json
 from typing import Dict, Any, Optional
 
+from edsl.exceptions import AgentRespondedWithBadJSONError
 from edsl.prompts.Prompt import Prompt
 from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
 from edsl.prompts.registry import get_classes as prompt_lookup
-from edsl.exceptions.questions import QuestionAnswerValidationError
-from edsl.agents.InvigilatorBase import InvigilatorBase
-from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
-from edsl.agents.PromptConstructor import PromptConstructor
-
+from edsl.data_transfer_models import AgentResponseDict
+from edsl.exceptions.agents import FailedTaskException
+from edsl.agents.PromptConstructionMixin import PromptConstructorMixin
 
-class NotApplicable(str):
-    def __new__(cls):
-        instance = super().__new__(cls, "Not Applicable")
-        instance.literal = "Not Applicable"
-        return instance
+from edsl.agents.InvigilatorBase import InvigilatorBase
 
 
-class InvigilatorAI(InvigilatorBase):
+class InvigilatorAI(PromptConstructorMixin, InvigilatorBase):
     """An invigilator that uses an AI model to answer questions."""
 
-    def get_prompts(self) -> Dict[str, Prompt]:
-        """Return the prompts used."""
-        return self.prompt_constructor.get_prompts()
-
     async def async_answer_question(self) -> AgentResponseDict:
         """Answer a question using the AI model.
 
         >>> i = InvigilatorAI.example()
         >>> i.answer_question()
-        {'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
+        {'message': '{"answer": "SPAM!"}'}
         """
-        prompts = self.get_prompts()
-        params = {
-            "user_prompt": prompts["user_prompt"].text,
-            "system_prompt": prompts["system_prompt"].text,
+        params = self.get_prompts() | {"iteration": self.iteration}
+        raw_response = await self.async_get_response(**params)
+        data = {
+            "agent": self.agent,
+            "question": self.question,
+            "scenario": self.scenario,
+            "raw_response": raw_response,
+            "raw_model_response": raw_response["raw_model_response"],
         }
-        if "encoded_image" in prompts:
-            params["encoded_image"] = prompts["encoded_image"]
-
-        params.update({"iteration": self.iteration, "cache": self.cache})
-
-        agent_response_dict: AgentResponseDict = await self.model.async_get_response(
-            **params
-        )
-        # store to self in case validation failure
-        self.raw_model_response = agent_response_dict.model_outputs.response
-        self.generated_tokens = agent_response_dict.edsl_dict.generated_tokens
-
-        return self.extract_edsl_result_entry_and_validate(agent_response_dict)
-
-    def _remove_from_cache(self, cache_key) -> None:
+        response = self._format_raw_response(**data)
+        # breakpoint()
+        return AgentResponseDict(**response)
+
+    async def async_get_response(
+        self,
+        user_prompt: Prompt,
+        system_prompt: Prompt,
+        iteration: int = 0,
+        encoded_image=None,
+    ) -> dict:
+        """Call the LLM and gets a response. Used in the `answer_question` method."""
+        try:
+            params = {
+                "user_prompt": user_prompt.text,
+                "system_prompt": system_prompt.text,
+                "iteration": iteration,
+                "cache": self.cache,
+            }
+            if encoded_image:
+                params["encoded_image"] = encoded_image
+            response = await self.model.async_get_response(**params)
+
+        # TODO: I *don't* think we need to delete the cache key here because I think
+        # it will not have been set yet; the exception would have been raised before.
+        except json.JSONDecodeError as e:
+            raise AgentRespondedWithBadJSONError(
+                f"Returned bad JSON: {e}"
+                f"Prompt: {user_prompt}"
+                f"System Prompt: {system_prompt}"
+            )
+
+        return response
+
+    def _remove_from_cache(self, raw_response) -> None:
         """Remove an entry from the cache."""
+        cache_key = raw_response.get("cache_key", None)
         if cache_key:
             del self.cache.data[cache_key]
 
-    def determine_answer(self, raw_answer: str) -> Any:
+    def _format_raw_response(
+        self, *, agent, question, scenario, raw_response, raw_model_response
+    ) -> AgentResponseDict:
+        """Return formatted raw response.
+
+        This cleans up the raw response to make it suitable to pass to AgentResponseDict.
+        """
+        _ = agent
+        try:
+            response = question._validate_answer(raw_response)
+        except Exception as e:
+            """If the response is invalid, remove it from the cache and raise the exception."""
+            self._remove_from_cache(raw_response)
+            raise e
+
         question_dict = self.survey.question_names_to_questions()
-        # iterates through the current answers and updates the question_dict (which is all questions)
         for other_question, answer in self.current_answers.items():
             if other_question in question_dict:
                 question_dict[other_question].answer = answer
             else:
-                # it might be a comment
+                # adds a comment to the question
                 if (
                     new_question := other_question.split("_comment")[0]
                 ) in question_dict:
                     question_dict[new_question].comment = answer
 
-        combined_dict = {**question_dict, **self.scenario}
-        # sometimes the answer is a code, so we need to translate it
-        return self.question._translate_answer_code_to_answer(raw_answer, combined_dict)
-
-    def extract_edsl_result_entry_and_validate(
-        self, agent_response_dict: AgentResponseDict
-    ) -> EDSLResultObjectInput:
-        edsl_dict = agent_response_dict.edsl_dict._asdict()
-        exception_occurred = None
-        validated = False
-        try:
-            validated_edsl_dict = self.question._validate_answer(edsl_dict)
-            answer = self.determine_answer(validated_edsl_dict["answer"])
-            comment = validated_edsl_dict.get("comment", "")
-            validated = True
-        except QuestionAnswerValidationError as e:
-            answer = None
-            comment = "The response was not valid."
-            if self.raise_validation_errors:
-                exception_occurred = e
-        except Exception as non_validation_error:
-            answer = None
-            comment = "Some other error occurred."
-            exception_occurred = non_validation_error
-        finally:
-            # even if validation failes, we still return the result
-            data = {
-                "answer": answer,
-                "comment": comment,
-                "generated_tokens": agent_response_dict.edsl_dict.generated_tokens,
-                "question_name": self.question.question_name,
-                "prompts": self.get_prompts(),
-                "cached_response": agent_response_dict.model_outputs.cached_response,
-                "raw_model_response": agent_response_dict.model_outputs.response,
-                "cache_used": agent_response_dict.model_outputs.cache_used,
-                "cache_key": agent_response_dict.model_outputs.cache_key,
-                "validated": validated,
-                "exception_occurred": exception_occurred,
-                "cost": agent_response_dict.model_outputs.cost,
-            }
-            result = EDSLResultObjectInput(**data)
-            return result
+        combined_dict = {**question_dict, **scenario}
+        answer = question._translate_answer_code_to_answer(
+            response["answer"], combined_dict
+        )
+        data = {
+            "answer": answer,
+            "comment": response.get(
+                "comment", ""
+            ),  # not all question have comment fields,
+            "question_name": question.question_name,
+            "prompts": self.get_prompts(),
+            "cached_response": raw_response.get("cached_response", None),
+            "usage": raw_response.get("usage", {}),
+            "raw_model_response": raw_model_response,
+            "cache_used": raw_response.get("cache_used", False),
+            "cache_key": raw_response.get("cache_key", None),
+        }
+        return AgentResponseDict(**data)
 
+    get_response = sync_wrapper(async_get_response)
     answer_question = sync_wrapper(async_answer_question)
 
 
-class InvigilatorHuman(InvigilatorBase):
-    """An invigilator for when a human is answering the question."""
+class InvigilatorSidecar(InvigilatorAI):
+    """An invigilator that presents the 'raw' question to the agent
+    & uses a sidecar model to answer questions."""
+
+    async def async_answer_question(self, failed: bool = False) -> AgentResponseDict:
+        """Answer a question using the AI model."""
+        from edsl import Model
+
+        advanced_model = self.sidecar_model
+        simple_model = self.model
+        question = self.question
+        human_readable_question = (
+            "Please answer this single question: " + question.human_readable()
+        )
+        print("Getting the simple model response to: ", human_readable_question)
+        raw_simple_response = await simple_model.async_execute_model_call(
+            user_prompt=human_readable_question,
+            system_prompt="""Pretend you are a human answering a question. Do not break character.""",
+        )
+        simple_response = simple_model.parse_response(raw_simple_response)
+        instructions = question.get_instructions()
+
+        main_model_prompt = Prompt(
+            text="""
+        A simpler language model was asked this question: 
+
+        To the simpel model:
+        {{ human_readable_question }}
+
+        The simple model responded:
+        <response>
+        {{ simple_response }}
+        </response>
+
+        It was suppose to respond according to these instructions:                                                      
+        <instructions>
+        {{ instructions }}
+        </instructions>
+                                
+        Please format the simple model's response as it should have been formmated, given the instructions.
+        Only respond in valid JSON, like so {"answer": "SPAM!"} or {"answer": "SPAM!", "comment": "I am a robot."}
+        Do not inlcude the word 'json'
+        """
+        )
+
+        d = {
+            "human_readable_question": human_readable_question,
+            "simple_response": simple_response,
+            "instructions": instructions,
+        }
+
+        print("The human-readable question is: ", human_readable_question)
+        print("The simple response is: ", simple_response)
+
+        raw_response_data = await advanced_model.async_execute_model_call(
+            user_prompt=main_model_prompt.render(d).text,
+            system_prompt="You are a helpful assistant.",
+        )
+
+        raw_response = await advanced_model.async_get_response(
+            user_prompt=main_model_prompt.render(d).text,
+            system_prompt="You are a helpful assistant.",
+            iteration=0,
+            cache=self.cache,
+        )
+
+        data = {
+            "agent": self.agent,
+            "question": self.question,
+            "scenario": self.scenario,
+        }
+        raw_response_data = {
+            "raw_response": raw_response,
+            "raw_model_response": raw_response["raw_model_response"],
+        }
+        params = data | raw_response_data
+        response = self._format_raw_response(**params)
+        response.update({"simple_model_raw_response": simple_response})
+        return AgentResponseDict(**response)
+
+    # get_response = sync_wrapper(async_get_response)
+    answer_question = sync_wrapper(async_answer_question)
 
-    validate_response: bool = False
-    translate_response: bool = False
+
+class InvigilatorDebug(InvigilatorBase):
+    """An invigilator class for debugging purposes."""
 
     async def async_answer_question(self, iteration: int = 0) -> AgentResponseDict:
         """Return the answer to the question."""
-        comment = "This is a real survey response from a human."
+        results = self.question._simulate_answer(human_readable=True)
+        results["prompts"] = self.get_prompts()
+        results["question_name"] = self.question.question_name
+        results["comment"] = "Debug comment"
+        return AgentResponseDict(**results)
+
+    def get_prompts(self) -> Dict[str, Prompt]:
+        """Return the prompts used."""
+        return {
+            "user_prompt": Prompt("NA"),
+            "system_prompt": Prompt("NA"),
+        }
 
-        def __repr__(self):
-            return f"{self.literal}"
 
-        exception_occurred = None
-        validated = False
+class InvigilatorHuman(InvigilatorBase):
+    """An invigilator for when a human is answering the question."""
+
+    async def async_answer_question(self, iteration: int = 0) -> AgentResponseDict:
+        """Return the answer to the question."""
+        data = {
+            "comment": "This is a real survey response from a human.",
+            "answer": None,
+            "prompts": self.get_prompts(),
+            "question_name": self.question.question_name,
+        }
         try:
             answer = self.agent.answer_question_directly(self.question, self.scenario)
-            self.raw_model_response = answer
-
-            if self.validate_response:
-                _ = self.question._validate_answer({"answer": answer})
-            if self.translate_response:
-                answer = self.question._translate_answer_code_to_answer(
-                    answer, self.scenario
-                )
-            validated = True
-        except QuestionAnswerValidationError as e:
-            answer = None
-            if self.raise_validation_errors:
-                exception_occurred = e
+            return AgentResponseDict(**(data | {"answer": answer}))
         except Exception as e:
-            answer = None
-            if self.raise_validation_errors:
-                exception_occurred = e
-        finally:
-            data = {
-                "generated_tokens": NotApplicable(),
-                "question_name": self.question.question_name,
-                "prompts": self.get_prompts(),
-                "cached_response": NotApplicable(),
-                "raw_model_response": NotApplicable(),
-                "cache_used": NotApplicable(),
-                "cache_key": NotApplicable(),
-                "answer": answer,
-                "comment": comment,
-                "validated": validated,
-                "exception_occurred": exception_occurred,
-            }
-            return EDSLResultObjectInput(**data)
+            agent_response_dict = AgentResponseDict(
+                **(data | {"answer": None, "comment": str(e)})
+            )
+            raise FailedTaskException(
+                f"Failed to get response. The exception is {str(e)}",
+                agent_response_dict,
+            ) from e
 
 
 class InvigilatorFunctional(InvigilatorBase):
@@ -172,21 +255,22 @@ class InvigilatorFunctional(InvigilatorBase):
     async def async_answer_question(self, iteration: int = 0) -> AgentResponseDict:
         """Return the answer to the question."""
         func = self.question.answer_question_directly
-        answer = func(scenario=self.scenario, agent_traits=self.agent.traits)
-
-        return EDSLResultObjectInput(
-            generated_tokens=str(answer),
-            question_name=self.question.question_name,
-            prompts=self.get_prompts(),
-            cached_response=NotApplicable(),
-            raw_model_response=NotApplicable(),
-            cache_used=NotApplicable(),
-            cache_key=NotApplicable(),
-            answer=answer["answer"],
-            comment="This is the result of a functional question.",
-            validated=True,
-            exception_occurred=None,
-        )
+        data = {
+            "comment": "Functional.",
+            "prompts": self.get_prompts(),
+            "question_name": self.question.question_name,
+        }
+        try:
+            answer = func(scenario=self.scenario, agent_traits=self.agent.traits)
+            return AgentResponseDict(**(data | answer))
+        except Exception as e:
+            agent_response_dict = AgentResponseDict(
+                **(data | {"answer": None, "comment": str(e)})
+            )
+            raise FailedTaskException(
+                f"Failed to get response. The exception is {str(e)}",
+                agent_response_dict,
+            ) from e
 
     def get_prompts(self) -> Dict[str, Prompt]:
         """Return the prompts used."""