edsl 0.1.33.dev1__py3-none-any.whl → 0.1.33.dev2__py3-none-any.whl

edsl/TemplateLoader.py

@@ -0,0 +1,24 @@
+from importlib import resources
+from jinja2 import BaseLoader, TemplateNotFound
+import os
+
+
+class TemplateLoader(BaseLoader):
+    def __init__(self, package_name, templates_dir):
+        self.package_name = package_name
+        self.templates_dir = templates_dir
+
+    def get_source(self, environment, template):
+        try:
+            parts = [self.templates_dir] + template.split("/")
+            template_path = os.path.join(*parts)
+
+            # Use resources.files() to get a Traversable object
+            templates = resources.files(self.package_name).joinpath(self.templates_dir)
+
+            # Use the read_text() method of the Traversable object
+            content = templates.joinpath(template).read_text()
+
+            return content, None, lambda: True
+        except FileNotFoundError:
+            raise TemplateNotFound(template)

edsl/__init__.py

@@ -8,9 +8,10 @@ from edsl.__version__ import __version__
 from edsl.config import Config, CONFIG
 from edsl.agents.Agent import Agent
 from edsl.agents.AgentList import AgentList
+
 from edsl.questions import QuestionBase
+from edsl.questions.question_registry import Question
 from edsl.questions import QuestionMultipleChoice
-from edsl.questions import QuestionBudget
 from edsl.questions import QuestionCheckBox
 from edsl.questions import QuestionExtract
 from edsl.questions import QuestionFreeText
@@ -19,10 +20,10 @@ from edsl.questions import QuestionLikertFive
 from edsl.questions import QuestionList
 from edsl.questions import QuestionLinearScale
 from edsl.questions import QuestionNumerical
-from edsl.questions import QuestionRank
-from edsl.questions import QuestionTopK
 from edsl.questions import QuestionYesNo
-from edsl.questions.question_registry import Question
+from edsl.questions import QuestionBudget
+from edsl.questions import QuestionRank
+
 from edsl.scenarios import Scenario
 from edsl.scenarios import ScenarioList
 
@@ -40,3 +41,6 @@ from edsl.notebooks.Notebook import Notebook
 from edsl.study.Study import Study
 from edsl.conjure.Conjure import Conjure
 from edsl.coop.coop import Coop
+
+from edsl.surveys.instructions.Instruction import Instruction
+from edsl.surveys.instructions.ChangeInstruction import ChangeInstruction

edsl/agents/Agent.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import copy
 import inspect
 import types
-from typing import Callable, Optional, Union
+from typing import Callable, Optional, Union, Any
 from uuid import uuid4
 from edsl.Base import Base
 
@@ -228,7 +228,12 @@ class Agent(Base):
         if hasattr(self, "answer_question_directly"):
             delattr(self, "answer_question_directly")
 
-    def add_direct_question_answering_method(self, method: Callable) -> None:
+    def add_direct_question_answering_method(
+        self,
+        method: Callable,
+        validate_response: bool = False,
+        translate_response: bool = False,
+    ) -> None:
         """Add a method to the agent that can answer a particular question type.
 
         :param method: A method that can answer a question directly.
@@ -249,6 +254,9 @@ class Agent(Base):
             )
             # print("Warning: overwriting existing answer_question_directly method")
 
+        self.validate_response = validate_response
+        self.translate_response = translate_response
+
         signature = inspect.signature(method)
         for argument in ["question", "scenario", "self"]:
             if argument not in signature.parameters:
@@ -272,6 +280,7 @@ class Agent(Base):
         current_answers: Optional[dict] = None,
         iteration: int = 1,
         sidecar_model=None,
+        raise_validation_errors: bool = True,
     ) -> "InvigilatorBase":
         """Create an Invigilator.
 
@@ -303,7 +312,12 @@ class Agent(Base):
             iteration=iteration,
             cache=cache,
             sidecar_model=sidecar_model,
+            raise_validation_errors=raise_validation_errors,
         )
+        if hasattr(self, "validate_response"):
+            invigilator.validate_response = self.validate_response
+        if hasattr(self, "translate_response"):
+            invigilator.translate_response = self.translate_response
         return invigilator
 
     async def async_answer_question(
@@ -334,8 +348,8 @@ class Agent(Base):
         >>> a.add_direct_question_answering_method(lambda self, question, scenario: "I am a direct answer.")
         >>> from edsl import QuestionFreeText
         >>> q = QuestionFreeText.example()
-        >>> a.answer_question(question = q, cache = False)
-        {'answer': 'I am a direct answer.', 'comment': 'This is a real survey response from a human.', ...}
+        >>> a.answer_question(question = q, cache = False).answer
+        'I am a direct answer.'
 
         This is a function where an agent returns an answer to a particular question.
         However, there are several different ways an agent can answer a question, so the
@@ -369,6 +383,7 @@ class Agent(Base):
         current_answers: Optional[dict] = None,
         iteration: int = 0,
         sidecar_model=None,
+        raise_validation_errors: bool = True,
     ) -> "InvigilatorBase":
         """Create an Invigilator."""
         from edsl import Model
@@ -378,7 +393,6 @@ class Agent(Base):
         scenario = scenario or Scenario()
 
         from edsl.agents.Invigilator import (
-            InvigilatorDebug,
             InvigilatorHuman,
             InvigilatorFunctional,
             InvigilatorAI,
@@ -391,8 +405,9 @@ class Agent(Base):
             cache = Cache()
 
         if debug:
+            raise NotImplementedError("Debug mode is not yet implemented.")
             # use the question's _simulate_answer method
-            invigilator_class = InvigilatorDebug
+            # invigilator_class = InvigilatorDebug
         elif hasattr(question, "answer_question_directly"):
             # It's a functional question and the answer only depends on the agent's traits & the scenario
             invigilator_class = InvigilatorFunctional
@@ -422,6 +437,7 @@ class Agent(Base):
             iteration=iteration,
             cache=cache,
             sidecar_model=sidecar_model,
+            raise_validation_errors=raise_validation_errors,
         )
         return invigilator
 
@@ -497,8 +513,8 @@ class Agent(Base):
         if name == "has_dynamic_traits_function":
             return self.has_dynamic_traits_function
 
-        if name in self.traits:
-            return self.traits[name]
+        if name in self._traits:
+            return self._traits[name]
         raise AttributeError(
             f"'{type(self).__name__}' object has no attribute '{name}'"
         )
@@ -570,9 +586,9 @@ class Agent(Base):
             if dynamic_traits_func:
                 func = inspect.getsource(dynamic_traits_func)
                 raw_data["dynamic_traits_function_source_code"] = func
-                raw_data[
-                    "dynamic_traits_function_name"
-                ] = self.dynamic_traits_function_name
+                raw_data["dynamic_traits_function_name"] = (
+                    self.dynamic_traits_function_name
+                )
         if hasattr(self, "answer_question_directly"):
             raw_data.pop(
                 "answer_question_directly", None
@@ -588,9 +604,9 @@ class Agent(Base):
                 raw_data["answer_question_directly_source_code"] = inspect.getsource(
                     answer_question_directly_func
                 )
-                raw_data[
-                    "answer_question_directly_function_name"
-                ] = self.answer_question_directly_function_name
+                raw_data["answer_question_directly_function_name"] = (
+                    self.answer_question_directly_function_name
+                )
 
         return raw_data
 
@@ -640,6 +656,22 @@ class Agent(Base):
         column_names = ["Attribute", "Value"]
         return table_data, column_names
 
+    def add_trait(self, trait_name_or_dict: str, value: Optional[Any] = None) -> Agent:
+        """Adds a trait to an agent and returns that agent"""
+        if isinstance(trait_name_or_dict, dict) and value is None:
+            self.traits.update(trait_name_or_dict)
+            return self
+
+        if isinstance(trait_name_or_dict, dict) and value:
+            raise ValueError(f"You passed a dict: {trait_name_or_dict}")
+
+        if isinstance(trait_name_or_dict, str):
+            trait = trait_name_or_dict
+            self.traits[trait] = value
+            return self
+
+        raise Exception("Something is not right with adding")
+
     def remove_trait(self, trait: str) -> Agent:
         """Remove a trait from the agent.
 

edsl/agents/AgentList.py

@@ -21,6 +21,12 @@ from simpleeval import EvalWithCompoundTypes
 from edsl.Base import Base
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
+from collections.abc import Iterable
+
+
+def is_iterable(obj):
+    return isinstance(obj, Iterable)
+
 
 class AgentList(UserList, Base):
     """A list of Agents."""
@@ -111,6 +117,13 @@ class AgentList(UserList, Base):
 
         return AgentList(new_data)
 
+    @property
+    def all_traits(self):
+        d = {}
+        for agent in self:
+            d.update(agent.traits)
+        return list(d.keys())
+
     @classmethod
     def from_csv(cls, file_path: str):
         """Load AgentList from a CSV file.
@@ -159,6 +172,36 @@ class AgentList(UserList, Base):
             _ = agent.remove_trait(trait)
         return self
 
+    def add_trait(self, trait, values):
+        """Adds a new trait to every agent, with values taken from values.
+
+        :param trait: The name of the trait.
+        :param values: The valeues(s) of the trait. If a single value is passed, it is used for all agents.
+
+        >>> al = AgentList.example()
+        >>> al.add_trait('new_trait', 1)
+        AgentList([Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5, 'new_trait': 1}), Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5, 'new_trait': 1})])
+        >>> al.select('new_trait').to_scenario_list().to_list()
+        [1, 1]
+        >>> al.add_trait('new_trait', [1, 2, 3])
+        Traceback (most recent call last):
+        ...
+        ValueError: The passed values have to be the same length as the agent list.
+        """
+        if not is_iterable(values):
+            value = values
+            for agent in self.data:
+                agent.add_trait(trait, value)
+            return self
+
+        if len(values) != len(self):
+            raise ValueError(
+                "The passed values have to be the same length as the agent list."
+            )
+        for agent, value in zip(self.data, values):
+            agent.add_trait(trait, value)
+        return self
+
     @staticmethod
     def get_codebook(file_path: str):
         """Return the codebook for a CSV file.

edsl/agents/Invigilator.py

@@ -1,17 +1,22 @@
 """Module for creating Invigilators, which are objects to administer a question to an Agent."""
 
-import json
 from typing import Dict, Any, Optional
 
 from edsl.exceptions import AgentRespondedWithBadJSONError
 from edsl.prompts.Prompt import Prompt
 from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
 from edsl.prompts.registry import get_classes as prompt_lookup
-from edsl.data_transfer_models import AgentResponseDict
-from edsl.exceptions.agents import FailedTaskException
+from edsl.exceptions.questions import QuestionAnswerValidationError
 from edsl.agents.PromptConstructionMixin import PromptConstructorMixin
-
 from edsl.agents.InvigilatorBase import InvigilatorBase
+from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
+
+
+class NotApplicable(str):
+    def __new__(cls):
+        instance = super().__new__(cls, "Not Applicable")
+        instance.literal = "Not Applicable"
+        return instance
 
 
 class InvigilatorAI(PromptConstructorMixin, InvigilatorBase):
@@ -22,231 +27,140 @@ class InvigilatorAI(PromptConstructorMixin, InvigilatorBase):
 
         >>> i = InvigilatorAI.example()
         >>> i.answer_question()
-        {'message': '{"answer": "SPAM!"}'}
+        {'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
         """
-        params = self.get_prompts() | {"iteration": self.iteration}
-        raw_response = await self.async_get_response(**params)
-        data = {
-            "agent": self.agent,
-            "question": self.question,
-            "scenario": self.scenario,
-            "raw_response": raw_response,
-            "raw_model_response": raw_response["raw_model_response"],
+        prompts = self.get_prompts()
+        params = {
+            "user_prompt": prompts["user_prompt"].text,
+            "system_prompt": prompts["system_prompt"].text,
         }
-        response = self._format_raw_response(**data)
-        # breakpoint()
-        return AgentResponseDict(**response)
+        if "encoded_image" in prompts:
+            params["encoded_image"] = prompts["encoded_image"]
 
-    async def async_get_response(
-        self,
-        user_prompt: Prompt,
-        system_prompt: Prompt,
-        iteration: int = 0,
-        encoded_image=None,
-    ) -> dict:
-        """Call the LLM and gets a response. Used in the `answer_question` method."""
-        try:
-            params = {
-                "user_prompt": user_prompt.text,
-                "system_prompt": system_prompt.text,
-                "iteration": iteration,
-                "cache": self.cache,
-            }
-            if encoded_image:
-                params["encoded_image"] = encoded_image
-            response = await self.model.async_get_response(**params)
+        params.update({"iteration": self.iteration, "cache": self.cache})
 
-        # TODO: I *don't* think we need to delete the cache key here because I think
-        # it will not have been set yet; the exception would have been raised before.
-        except json.JSONDecodeError as e:
-            raise AgentRespondedWithBadJSONError(
-                f"Returned bad JSON: {e}"
-                f"Prompt: {user_prompt}"
-                f"System Prompt: {system_prompt}"
-            )
+        agent_response_dict: AgentResponseDict = await self.model.async_get_response(
+            **params
+        )
+        # store to self in case validation failure
+        self.raw_model_response = agent_response_dict.model_outputs.response
+        self.generated_tokens = agent_response_dict.edsl_dict.generated_tokens
 
-        return response
+        return self.extract_edsl_result_entry_and_validate(agent_response_dict)
 
-    def _remove_from_cache(self, raw_response) -> None:
+    def _remove_from_cache(self, cache_key) -> None:
         """Remove an entry from the cache."""
-        cache_key = raw_response.get("cache_key", None)
         if cache_key:
             del self.cache.data[cache_key]
 
-    def _format_raw_response(
-        self, *, agent, question, scenario, raw_response, raw_model_response
-    ) -> AgentResponseDict:
-        """Return formatted raw response.
-
-        This cleans up the raw response to make it suitable to pass to AgentResponseDict.
-        """
-        _ = agent
-        try:
-            response = question._validate_answer(raw_response)
-        except Exception as e:
-            """If the response is invalid, remove it from the cache and raise the exception."""
-            self._remove_from_cache(raw_response)
-            raise e
-
+    def determine_answer(self, raw_answer: str) -> Any:
         question_dict = self.survey.question_names_to_questions()
+        # iterates through the current answers and updates the question_dict (which is all questions)
         for other_question, answer in self.current_answers.items():
             if other_question in question_dict:
                 question_dict[other_question].answer = answer
             else:
-                # adds a comment to the question
+                # it might be a comment
                 if (
                     new_question := other_question.split("_comment")[0]
                 ) in question_dict:
                     question_dict[new_question].comment = answer
 
-        combined_dict = {**question_dict, **scenario}
-        answer = question._translate_answer_code_to_answer(
-            response["answer"], combined_dict
-        )
-        data = {
-            "answer": answer,
-            "comment": response.get(
-                "comment", ""
-            ),  # not all question have comment fields,
-            "question_name": question.question_name,
-            "prompts": self.get_prompts(),
-            "cached_response": raw_response.get("cached_response", None),
-            "usage": raw_response.get("usage", {}),
-            "raw_model_response": raw_model_response,
-            "cache_used": raw_response.get("cache_used", False),
-            "cache_key": raw_response.get("cache_key", None),
-        }
-        return AgentResponseDict(**data)
-
-    get_response = sync_wrapper(async_get_response)
-    answer_question = sync_wrapper(async_answer_question)
-
-
-class InvigilatorSidecar(InvigilatorAI):
-    """An invigilator that presents the 'raw' question to the agent
-    & uses a sidecar model to answer questions."""
-
-    async def async_answer_question(self, failed: bool = False) -> AgentResponseDict:
-        """Answer a question using the AI model."""
-        from edsl import Model
-
-        advanced_model = self.sidecar_model
-        simple_model = self.model
-        question = self.question
-        human_readable_question = (
-            "Please answer this single question: " + question.human_readable()
-        )
-        print("Getting the simple model response to: ", human_readable_question)
-        raw_simple_response = await simple_model.async_execute_model_call(
-            user_prompt=human_readable_question,
-            system_prompt="""Pretend you are a human answering a question. Do not break character.""",
-        )
-        simple_response = simple_model.parse_response(raw_simple_response)
-        instructions = question.get_instructions()
-
-        main_model_prompt = Prompt(
-            text="""
-        A simpler language model was asked this question: 
-
-        To the simpel model:
-        {{ human_readable_question }}
-
-        The simple model responded:
-        <response>
-        {{ simple_response }}
-        </response>
-
-        It was suppose to respond according to these instructions:                                                      
-        <instructions>
-        {{ instructions }}
-        </instructions>
-                                
-        Please format the simple model's response as it should have been formmated, given the instructions.
-        Only respond in valid JSON, like so {"answer": "SPAM!"} or {"answer": "SPAM!", "comment": "I am a robot."}
-        Do not inlcude the word 'json'
-        """
-        )
-
-        d = {
-            "human_readable_question": human_readable_question,
-            "simple_response": simple_response,
-            "instructions": instructions,
-        }
-
-        print("The human-readable question is: ", human_readable_question)
-        print("The simple response is: ", simple_response)
+        combined_dict = {**question_dict, **self.scenario}
+        # sometimes the answer is a code, so we need to translate it
+        return self.question._translate_answer_code_to_answer(raw_answer, combined_dict)
 
-        raw_response_data = await advanced_model.async_execute_model_call(
-            user_prompt=main_model_prompt.render(d).text,
-            system_prompt="You are a helpful assistant.",
-        )
-
-        raw_response = await advanced_model.async_get_response(
-            user_prompt=main_model_prompt.render(d).text,
-            system_prompt="You are a helpful assistant.",
-            iteration=0,
-            cache=self.cache,
-        )
-
-        data = {
-            "agent": self.agent,
-            "question": self.question,
-            "scenario": self.scenario,
-        }
-        raw_response_data = {
-            "raw_response": raw_response,
-            "raw_model_response": raw_response["raw_model_response"],
-        }
-        params = data | raw_response_data
-        response = self._format_raw_response(**params)
-        response.update({"simple_model_raw_response": simple_response})
-        return AgentResponseDict(**response)
+    def extract_edsl_result_entry_and_validate(
+        self, agent_response_dict: AgentResponseDict
+    ) -> EDSLResultObjectInput:
+        edsl_dict = agent_response_dict.edsl_dict._asdict()
+        exception_occurred = None
+        validated = False
+        try:
+            validated_edsl_dict = self.question._validate_answer(edsl_dict)
+            answer = self.determine_answer(validated_edsl_dict["answer"])
+            comment = validated_edsl_dict.get("comment", "")
+            validated = True
+        except QuestionAnswerValidationError as e:
+            answer = None
+            comment = "The response was not valid."
+            if self.raise_validation_errors:
+                exception_occurred = e
+        except Exception as non_validation_error:
+            answer = None
+            comment = "Some other error occurred."
+            exception_occurred = non_validation_error
+        finally:
+            # even if validation failes, we still return the result
+            data = {
+                "answer": answer,
+                "comment": comment,
+                "generated_tokens": agent_response_dict.edsl_dict.generated_tokens,
+                "question_name": self.question.question_name,
+                "prompts": self.get_prompts(),
+                "cached_response": agent_response_dict.model_outputs.cached_response,
+                "raw_model_response": agent_response_dict.model_outputs.response,
+                "cache_used": agent_response_dict.model_outputs.cache_used,
+                "cache_key": agent_response_dict.model_outputs.cache_key,
+                "validated": validated,
+                "exception_occurred": exception_occurred,
+                "cost": agent_response_dict.model_outputs.cost,
+            }
+            result = EDSLResultObjectInput(**data)
+            return result
 
-    # get_response = sync_wrapper(async_get_response)
     answer_question = sync_wrapper(async_answer_question)
 
 
-class InvigilatorDebug(InvigilatorBase):
-    """An invigilator class for debugging purposes."""
-
-    async def async_answer_question(self, iteration: int = 0) -> AgentResponseDict:
-        """Return the answer to the question."""
-        results = self.question._simulate_answer(human_readable=True)
-        results["prompts"] = self.get_prompts()
-        results["question_name"] = self.question.question_name
-        results["comment"] = "Debug comment"
-        return AgentResponseDict(**results)
-
-    def get_prompts(self) -> Dict[str, Prompt]:
-        """Return the prompts used."""
-        return {
-            "user_prompt": Prompt("NA"),
-            "system_prompt": Prompt("NA"),
-        }
-
-
 class InvigilatorHuman(InvigilatorBase):
     """An invigilator for when a human is answering the question."""
 
+    validate_response: bool = False
+    translate_response: bool = False
+
     async def async_answer_question(self, iteration: int = 0) -> AgentResponseDict:
         """Return the answer to the question."""
-        data = {
-            "comment": "This is a real survey response from a human.",
-            "answer": None,
-            "prompts": self.get_prompts(),
-            "question_name": self.question.question_name,
-        }
+        comment = "This is a real survey response from a human."
+
+        def __repr__(self):
+            return f"{self.literal}"
+
+        exception_occurred = None
+        validated = False
         try:
             answer = self.agent.answer_question_directly(self.question, self.scenario)
-            return AgentResponseDict(**(data | {"answer": answer}))
+            self.raw_model_response = answer
+
+            if self.validate_response:
+                _ = self.question._validate_answer({"answer": answer})
+            if self.translate_response:
+                answer = self.question._translate_answer_code_to_answer(
+                    answer, self.scenario
+                )
+            validated = True
+        except QuestionAnswerValidationError as e:
+            answer = None
+            if self.raise_validation_errors:
+                exception_occurred = e
         except Exception as e:
-            agent_response_dict = AgentResponseDict(
-                **(data | {"answer": None, "comment": str(e)})
-            )
-            raise FailedTaskException(
-                f"Failed to get response. The exception is {str(e)}",
-                agent_response_dict,
-            ) from e
+            answer = None
+            if self.raise_validation_errors:
+                exception_occurred = e
+        finally:
+            data = {
+                "generated_tokens": NotApplicable(),
+                "question_name": self.question.question_name,
+                "prompts": self.get_prompts(),
+                "cached_response": NotApplicable(),
+                "raw_model_response": NotApplicable(),
+                "cache_used": NotApplicable(),
+                "cache_key": NotApplicable(),
+                "answer": answer,
+                "comment": comment,
+                "validated": validated,
+                "exception_occurred": exception_occurred,
+            }
+            return EDSLResultObjectInput(**data)
 
 
 class InvigilatorFunctional(InvigilatorBase):
@@ -255,22 +169,21 @@ class InvigilatorFunctional(InvigilatorBase):
     async def async_answer_question(self, iteration: int = 0) -> AgentResponseDict:
         """Return the answer to the question."""
         func = self.question.answer_question_directly
-        data = {
-            "comment": "Functional.",
-            "prompts": self.get_prompts(),
-            "question_name": self.question.question_name,
-        }
-        try:
-            answer = func(scenario=self.scenario, agent_traits=self.agent.traits)
-            return AgentResponseDict(**(data | answer))
-        except Exception as e:
-            agent_response_dict = AgentResponseDict(
-                **(data | {"answer": None, "comment": str(e)})
-            )
-            raise FailedTaskException(
-                f"Failed to get response. The exception is {str(e)}",
-                agent_response_dict,
-            ) from e
+        answer = func(scenario=self.scenario, agent_traits=self.agent.traits)
+
+        return EDSLResultObjectInput(
+            generated_tokens=str(answer),
+            question_name=self.question.question_name,
+            prompts=self.get_prompts(),
+            cached_response=NotApplicable(),
+            raw_model_response=NotApplicable(),
+            cache_used=NotApplicable(),
+            cache_key=NotApplicable(),
+            answer=answer["answer"],
+            comment="This is the result of a functional question.",
+            validated=True,
+            exception_occurred=None,
+        )
 
     def get_prompts(self) -> Dict[str, Prompt]:
         """Return the prompts used."""