edsl 0.1.33.dev1__tar.gz → 0.1.33.dev2__tar.gz

{edsl-0.1.33.dev1 → edsl-0.1.33.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edsl
-Version: 0.1.33.dev1
+Version: 0.1.33.dev2
 Summary: Create and analyze LLM-based surveys
 Home-page: https://www.expectedparrot.com/
 License: MIT
@@ -23,9 +23,11 @@ Requires-Dist: black[jupyter] (>=24.4.2,<25.0.0)
 Requires-Dist: boto3 (>=1.34.161,<2.0.0)
 Requires-Dist: groq (>=0.9.0,<0.10.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
+Requires-Dist: json-repair (>=0.28.4,<0.29.0)
 Requires-Dist: jupyter (>=1.0.0,<2.0.0)
 Requires-Dist: markdown2 (>=2.4.11,<3.0.0)
 Requires-Dist: matplotlib (>=3.8,<3.9)
+Requires-Dist: mistralai (>=1.0.2,<2.0.0)
 Requires-Dist: nest-asyncio (>=1.5.9,<2.0.0)
 Requires-Dist: numpy (>=1.22,<2.0)
 Requires-Dist: openai (>=1.4.0,<2.0.0)
@@ -67,7 +69,7 @@ A quick example:
 
 ```python
 # Import a question type
-from edsl.questions import QuestionMultipleChoice
+from edsl import QuestionMultipleChoice
 
 # Construct a question using the question type template
 q = QuestionMultipleChoice(

{edsl-0.1.33.dev1 → edsl-0.1.33.dev2}/README.md

@@ -19,7 +19,7 @@ A quick example:
 
 ```python
 # Import a question type
-from edsl.questions import QuestionMultipleChoice
+from edsl import QuestionMultipleChoice
 
 # Construct a question using the question type template
 q = QuestionMultipleChoice(

edsl-0.1.33.dev2/edsl/TemplateLoader.py

@@ -0,0 +1,24 @@
+from importlib import resources
+from jinja2 import BaseLoader, TemplateNotFound
+import os
+
+
+class TemplateLoader(BaseLoader):
+    def __init__(self, package_name, templates_dir):
+        self.package_name = package_name
+        self.templates_dir = templates_dir
+
+    def get_source(self, environment, template):
+        try:
+            parts = [self.templates_dir] + template.split("/")
+            template_path = os.path.join(*parts)
+
+            # Use resources.files() to get a Traversable object
+            templates = resources.files(self.package_name).joinpath(self.templates_dir)
+
+            # Use the read_text() method of the Traversable object
+            content = templates.joinpath(template).read_text()
+
+            return content, None, lambda: True
+        except FileNotFoundError:
+            raise TemplateNotFound(template)

{edsl-0.1.33.dev1 → edsl-0.1.33.dev2}/edsl/__init__.py

@@ -8,9 +8,10 @@ from edsl.__version__ import __version__
 from edsl.config import Config, CONFIG
 from edsl.agents.Agent import Agent
 from edsl.agents.AgentList import AgentList
+
 from edsl.questions import QuestionBase
+from edsl.questions.question_registry import Question
 from edsl.questions import QuestionMultipleChoice
-from edsl.questions import QuestionBudget
 from edsl.questions import QuestionCheckBox
 from edsl.questions import QuestionExtract
 from edsl.questions import QuestionFreeText
@@ -19,10 +20,10 @@ from edsl.questions import QuestionLikertFive
 from edsl.questions import QuestionList
 from edsl.questions import QuestionLinearScale
 from edsl.questions import QuestionNumerical
-from edsl.questions import QuestionRank
-from edsl.questions import QuestionTopK
 from edsl.questions import QuestionYesNo
-from edsl.questions.question_registry import Question
+from edsl.questions import QuestionBudget
+from edsl.questions import QuestionRank
+
 from edsl.scenarios import Scenario
 from edsl.scenarios import ScenarioList
 
@@ -40,3 +41,6 @@ from edsl.notebooks.Notebook import Notebook
 from edsl.study.Study import Study
 from edsl.conjure.Conjure import Conjure
 from edsl.coop.coop import Coop
+
+from edsl.surveys.instructions.Instruction import Instruction
+from edsl.surveys.instructions.ChangeInstruction import ChangeInstruction

{edsl-0.1.33.dev1 → edsl-0.1.33.dev2}/edsl/agents/Agent.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import copy
 import inspect
 import types
-from typing import Callable, Optional, Union
+from typing import Callable, Optional, Union, Any
 from uuid import uuid4
 from edsl.Base import Base
 
@@ -228,7 +228,12 @@ class Agent(Base):
         if hasattr(self, "answer_question_directly"):
             delattr(self, "answer_question_directly")
 
-    def add_direct_question_answering_method(self, method: Callable) -> None:
+    def add_direct_question_answering_method(
+        self,
+        method: Callable,
+        validate_response: bool = False,
+        translate_response: bool = False,
+    ) -> None:
         """Add a method to the agent that can answer a particular question type.
 
         :param method: A method that can answer a question directly.
@@ -249,6 +254,9 @@ class Agent(Base):
             )
             # print("Warning: overwriting existing answer_question_directly method")
 
+        self.validate_response = validate_response
+        self.translate_response = translate_response
+
         signature = inspect.signature(method)
         for argument in ["question", "scenario", "self"]:
             if argument not in signature.parameters:
@@ -272,6 +280,7 @@ class Agent(Base):
         current_answers: Optional[dict] = None,
         iteration: int = 1,
         sidecar_model=None,
+        raise_validation_errors: bool = True,
     ) -> "InvigilatorBase":
         """Create an Invigilator.
 
@@ -303,7 +312,12 @@ class Agent(Base):
             iteration=iteration,
             cache=cache,
             sidecar_model=sidecar_model,
+            raise_validation_errors=raise_validation_errors,
         )
+        if hasattr(self, "validate_response"):
+            invigilator.validate_response = self.validate_response
+        if hasattr(self, "translate_response"):
+            invigilator.translate_response = self.translate_response
         return invigilator
 
     async def async_answer_question(
@@ -334,8 +348,8 @@ class Agent(Base):
         >>> a.add_direct_question_answering_method(lambda self, question, scenario: "I am a direct answer.")
         >>> from edsl import QuestionFreeText
         >>> q = QuestionFreeText.example()
-        >>> a.answer_question(question = q, cache = False)
-        {'answer': 'I am a direct answer.', 'comment': 'This is a real survey response from a human.', ...}
+        >>> a.answer_question(question = q, cache = False).answer
+        'I am a direct answer.'
 
         This is a function where an agent returns an answer to a particular question.
         However, there are several different ways an agent can answer a question, so the
@@ -369,6 +383,7 @@ class Agent(Base):
         current_answers: Optional[dict] = None,
         iteration: int = 0,
         sidecar_model=None,
+        raise_validation_errors: bool = True,
     ) -> "InvigilatorBase":
         """Create an Invigilator."""
         from edsl import Model
@@ -378,7 +393,6 @@ class Agent(Base):
         scenario = scenario or Scenario()
 
         from edsl.agents.Invigilator import (
-            InvigilatorDebug,
             InvigilatorHuman,
             InvigilatorFunctional,
             InvigilatorAI,
@@ -391,8 +405,9 @@ class Agent(Base):
             cache = Cache()
 
         if debug:
+            raise NotImplementedError("Debug mode is not yet implemented.")
             # use the question's _simulate_answer method
-            invigilator_class = InvigilatorDebug
+            # invigilator_class = InvigilatorDebug
         elif hasattr(question, "answer_question_directly"):
             # It's a functional question and the answer only depends on the agent's traits & the scenario
             invigilator_class = InvigilatorFunctional
@@ -422,6 +437,7 @@ class Agent(Base):
             iteration=iteration,
             cache=cache,
             sidecar_model=sidecar_model,
+            raise_validation_errors=raise_validation_errors,
         )
         return invigilator
 
@@ -497,8 +513,8 @@ class Agent(Base):
         if name == "has_dynamic_traits_function":
             return self.has_dynamic_traits_function
 
-        if name in self.traits:
-            return self.traits[name]
+        if name in self._traits:
+            return self._traits[name]
         raise AttributeError(
             f"'{type(self).__name__}' object has no attribute '{name}'"
         )
@@ -570,9 +586,9 @@ class Agent(Base):
             if dynamic_traits_func:
                 func = inspect.getsource(dynamic_traits_func)
                 raw_data["dynamic_traits_function_source_code"] = func
-                raw_data[
-                    "dynamic_traits_function_name"
-                ] = self.dynamic_traits_function_name
+                raw_data["dynamic_traits_function_name"] = (
+                    self.dynamic_traits_function_name
+                )
         if hasattr(self, "answer_question_directly"):
             raw_data.pop(
                 "answer_question_directly", None
@@ -588,9 +604,9 @@ class Agent(Base):
                 raw_data["answer_question_directly_source_code"] = inspect.getsource(
                     answer_question_directly_func
                 )
-                raw_data[
-                    "answer_question_directly_function_name"
-                ] = self.answer_question_directly_function_name
+                raw_data["answer_question_directly_function_name"] = (
+                    self.answer_question_directly_function_name
+                )
 
         return raw_data
 
@@ -640,6 +656,22 @@ class Agent(Base):
         column_names = ["Attribute", "Value"]
         return table_data, column_names
 
+    def add_trait(self, trait_name_or_dict: str, value: Optional[Any] = None) -> Agent:
+        """Adds a trait to an agent and returns that agent"""
+        if isinstance(trait_name_or_dict, dict) and value is None:
+            self.traits.update(trait_name_or_dict)
+            return self
+
+        if isinstance(trait_name_or_dict, dict) and value:
+            raise ValueError(f"You passed a dict: {trait_name_or_dict}")
+
+        if isinstance(trait_name_or_dict, str):
+            trait = trait_name_or_dict
+            self.traits[trait] = value
+            return self
+
+        raise Exception("Something is not right with adding")
+
     def remove_trait(self, trait: str) -> Agent:
         """Remove a trait from the agent.
 

{edsl-0.1.33.dev1 → edsl-0.1.33.dev2}/edsl/agents/AgentList.py

@@ -21,6 +21,12 @@ from simpleeval import EvalWithCompoundTypes
 from edsl.Base import Base
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
+from collections.abc import Iterable
+
+
+def is_iterable(obj):
+    return isinstance(obj, Iterable)
+
 
 class AgentList(UserList, Base):
     """A list of Agents."""
@@ -111,6 +117,13 @@ class AgentList(UserList, Base):
 
         return AgentList(new_data)
 
+    @property
+    def all_traits(self):
+        d = {}
+        for agent in self:
+            d.update(agent.traits)
+        return list(d.keys())
+
     @classmethod
     def from_csv(cls, file_path: str):
         """Load AgentList from a CSV file.
@@ -159,6 +172,36 @@ class AgentList(UserList, Base):
             _ = agent.remove_trait(trait)
         return self
 
+    def add_trait(self, trait, values):
+        """Adds a new trait to every agent, with values taken from values.
+
+        :param trait: The name of the trait.
+        :param values: The valeues(s) of the trait. If a single value is passed, it is used for all agents.
+
+        >>> al = AgentList.example()
+        >>> al.add_trait('new_trait', 1)
+        AgentList([Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5, 'new_trait': 1}), Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5, 'new_trait': 1})])
+        >>> al.select('new_trait').to_scenario_list().to_list()
+        [1, 1]
+        >>> al.add_trait('new_trait', [1, 2, 3])
+        Traceback (most recent call last):
+        ...
+        ValueError: The passed values have to be the same length as the agent list.
+        """
+        if not is_iterable(values):
+            value = values
+            for agent in self.data:
+                agent.add_trait(trait, value)
+            return self
+
+        if len(values) != len(self):
+            raise ValueError(
+                "The passed values have to be the same length as the agent list."
+            )
+        for agent, value in zip(self.data, values):
+            agent.add_trait(trait, value)
+        return self
+
     @staticmethod
     def get_codebook(file_path: str):
         """Return the codebook for a CSV file.

edsl-0.1.33.dev2/edsl/agents/Invigilator.py

@@ -0,0 +1,199 @@
+"""Module for creating Invigilators, which are objects to administer a question to an Agent."""
+
+from typing import Dict, Any, Optional
+
+from edsl.exceptions import AgentRespondedWithBadJSONError
+from edsl.prompts.Prompt import Prompt
+from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
+from edsl.prompts.registry import get_classes as prompt_lookup
+from edsl.exceptions.questions import QuestionAnswerValidationError
+from edsl.agents.PromptConstructionMixin import PromptConstructorMixin
+from edsl.agents.InvigilatorBase import InvigilatorBase
+from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
+
+
+class NotApplicable(str):
+    def __new__(cls):
+        instance = super().__new__(cls, "Not Applicable")
+        instance.literal = "Not Applicable"
+        return instance
+
+
+class InvigilatorAI(PromptConstructorMixin, InvigilatorBase):
+    """An invigilator that uses an AI model to answer questions."""
+
+    async def async_answer_question(self) -> AgentResponseDict:
+        """Answer a question using the AI model.
+
+        >>> i = InvigilatorAI.example()
+        >>> i.answer_question()
+        {'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
+        """
+        prompts = self.get_prompts()
+        params = {
+            "user_prompt": prompts["user_prompt"].text,
+            "system_prompt": prompts["system_prompt"].text,
+        }
+        if "encoded_image" in prompts:
+            params["encoded_image"] = prompts["encoded_image"]
+
+        params.update({"iteration": self.iteration, "cache": self.cache})
+
+        agent_response_dict: AgentResponseDict = await self.model.async_get_response(
+            **params
+        )
+        # store to self in case validation failure
+        self.raw_model_response = agent_response_dict.model_outputs.response
+        self.generated_tokens = agent_response_dict.edsl_dict.generated_tokens
+
+        return self.extract_edsl_result_entry_and_validate(agent_response_dict)
+
+    def _remove_from_cache(self, cache_key) -> None:
+        """Remove an entry from the cache."""
+        if cache_key:
+            del self.cache.data[cache_key]
+
+    def determine_answer(self, raw_answer: str) -> Any:
+        question_dict = self.survey.question_names_to_questions()
+        # iterates through the current answers and updates the question_dict (which is all questions)
+        for other_question, answer in self.current_answers.items():
+            if other_question in question_dict:
+                question_dict[other_question].answer = answer
+            else:
+                # it might be a comment
+                if (
+                    new_question := other_question.split("_comment")[0]
+                ) in question_dict:
+                    question_dict[new_question].comment = answer
+
+        combined_dict = {**question_dict, **self.scenario}
+        # sometimes the answer is a code, so we need to translate it
+        return self.question._translate_answer_code_to_answer(raw_answer, combined_dict)
+
+    def extract_edsl_result_entry_and_validate(
+        self, agent_response_dict: AgentResponseDict
+    ) -> EDSLResultObjectInput:
+        edsl_dict = agent_response_dict.edsl_dict._asdict()
+        exception_occurred = None
+        validated = False
+        try:
+            validated_edsl_dict = self.question._validate_answer(edsl_dict)
+            answer = self.determine_answer(validated_edsl_dict["answer"])
+            comment = validated_edsl_dict.get("comment", "")
+            validated = True
+        except QuestionAnswerValidationError as e:
+            answer = None
+            comment = "The response was not valid."
+            if self.raise_validation_errors:
+                exception_occurred = e
+        except Exception as non_validation_error:
+            answer = None
+            comment = "Some other error occurred."
+            exception_occurred = non_validation_error
+        finally:
+            # even if validation failes, we still return the result
+            data = {
+                "answer": answer,
+                "comment": comment,
+                "generated_tokens": agent_response_dict.edsl_dict.generated_tokens,
+                "question_name": self.question.question_name,
+                "prompts": self.get_prompts(),
+                "cached_response": agent_response_dict.model_outputs.cached_response,
+                "raw_model_response": agent_response_dict.model_outputs.response,
+                "cache_used": agent_response_dict.model_outputs.cache_used,
+                "cache_key": agent_response_dict.model_outputs.cache_key,
+                "validated": validated,
+                "exception_occurred": exception_occurred,
+                "cost": agent_response_dict.model_outputs.cost,
+            }
+            result = EDSLResultObjectInput(**data)
+            return result
+
+    answer_question = sync_wrapper(async_answer_question)
+
+
+class InvigilatorHuman(InvigilatorBase):
+    """An invigilator for when a human is answering the question."""
+
+    validate_response: bool = False
+    translate_response: bool = False
+
+    async def async_answer_question(self, iteration: int = 0) -> AgentResponseDict:
+        """Return the answer to the question."""
+        comment = "This is a real survey response from a human."
+
+        def __repr__(self):
+            return f"{self.literal}"
+
+        exception_occurred = None
+        validated = False
+        try:
+            answer = self.agent.answer_question_directly(self.question, self.scenario)
+            self.raw_model_response = answer
+
+            if self.validate_response:
+                _ = self.question._validate_answer({"answer": answer})
+            if self.translate_response:
+                answer = self.question._translate_answer_code_to_answer(
+                    answer, self.scenario
+                )
+            validated = True
+        except QuestionAnswerValidationError as e:
+            answer = None
+            if self.raise_validation_errors:
+                exception_occurred = e
+        except Exception as e:
+            answer = None
+            if self.raise_validation_errors:
+                exception_occurred = e
+        finally:
+            data = {
+                "generated_tokens": NotApplicable(),
+                "question_name": self.question.question_name,
+                "prompts": self.get_prompts(),
+                "cached_response": NotApplicable(),
+                "raw_model_response": NotApplicable(),
+                "cache_used": NotApplicable(),
+                "cache_key": NotApplicable(),
+                "answer": answer,
+                "comment": comment,
+                "validated": validated,
+                "exception_occurred": exception_occurred,
+            }
+            return EDSLResultObjectInput(**data)
+
+
+class InvigilatorFunctional(InvigilatorBase):
+    """A Invigilator for when the question has a answer_question_directly function."""
+
+    async def async_answer_question(self, iteration: int = 0) -> AgentResponseDict:
+        """Return the answer to the question."""
+        func = self.question.answer_question_directly
+        answer = func(scenario=self.scenario, agent_traits=self.agent.traits)
+
+        return EDSLResultObjectInput(
+            generated_tokens=str(answer),
+            question_name=self.question.question_name,
+            prompts=self.get_prompts(),
+            cached_response=NotApplicable(),
+            raw_model_response=NotApplicable(),
+            cache_used=NotApplicable(),
+            cache_key=NotApplicable(),
+            answer=answer["answer"],
+            comment="This is the result of a functional question.",
+            validated=True,
+            exception_occurred=None,
+        )
+
+    def get_prompts(self) -> Dict[str, Prompt]:
+        """Return the prompts used."""
+        return {
+            "user_prompt": Prompt("NA"),
+            "system_prompt": Prompt("NA"),
+        }
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)