edsl 0.1.15__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/questions/QuestionFreeText.py

@@ -1,89 +1,113 @@
 from __future__ import annotations
-import textwrap
 from typing import Any, Optional
-from edsl.questions import Question
-from edsl.questions.descriptors import AllowNonresponseDescriptor
-from edsl.scenarios import Scenario
-from edsl.utilities import random_string
+from uuid import uuid4
 
+from pydantic import field_validator
 
-# TODO: should allow answer = {"answer": None} if allow_nonresponse is True
-class QuestionFreeText(Question):
-    """
-    This question asks the user to answer with free text.
+from edsl.questions.QuestionBase import QuestionBase
+from edsl.questions.response_validator_abc import ResponseValidatorABC
+
+from edsl.exceptions.questions import QuestionAnswerValidationError
+from edsl.questions.decorators import inject_exception
+
+from pydantic import BaseModel
+from typing import Optional, Any, List
 
-    Arguments:
-    - `question_name` is the name of the question (string)
-    - `question_text` is the text of the question (string)
+from edsl.prompts.Prompt import Prompt
 
-    Optional arguments:
-    - `allow_nonresponse` is whether the user can skip the question (boolean). If not provided, the default is False.
-    - `instructions` are the instructions for the question (string). If not provided, the default instructions are used. To view them, run `QuestionFreeText.default_instructions`
 
-    For an example, run `QuestionFreeText.example()`
+class FreeTextResponse(BaseModel):
     """
+    Validator for free text response questions.
+    """
+
+    answer: str
+    generated_tokens: Optional[str] = None
+
+
+class FreeTextResponseValidator(ResponseValidatorABC):
+    required_params = []
+    valid_examples = [({"answer": "This is great"}, {})]
+    invalid_examples = [
+        (
+            {"answer": None},
+            {},
+            "Answer code must not be missing.",
+        ),
+    ]
+
+    def fix(self, response, verbose=False):
+        return {
+            "answer": str(response.get("generated_tokens")),
+            "generated_tokens": str(response.get("generated_tokens")),
+        }
+
+
+class QuestionFreeText(QuestionBase):
+    """This question prompts the agent to respond with free text."""
 
     question_type = "free_text"
-    allow_nonresponse: bool = AllowNonresponseDescriptor()
-    default_instructions = textwrap.dedent(
-        """\
-        You are being asked the following question: {{question_text}}
-        Return a valid JSON formatted like this: 
-        {"answer": "<put free text answer here>"}
-        """
-    )
+    _response_model = FreeTextResponse
+    response_validator_class = FreeTextResponseValidator
 
     def __init__(
         self,
-        question_text: str,
         question_name: str,
-        allow_nonresponse: Optional[bool] = None,
-        instructions: Optional[str] = None,
+        question_text: str,
+        answering_instructions: Optional[str] = None,
+        question_presentation: Optional[str] = None,
     ):
-        self.question_text = question_text
-        self.question_name = question_name
-        self.allow_nonresponse = allow_nonresponse or False
-        self.instructions = instructions
+        """Instantiate a new QuestionFreeText.
 
-    ################
-    # Answer methods
-    ################
-    def validate_answer(self, answer: Any) -> dict[str, str]:
-        self.validate_answer_template_basic(answer)
-        self.validate_answer_key_value(answer, "answer", str)
-        return answer
+        :param question_name: The name of the question.
+        :param question_text: The text of the question.
+        """
+        self.question_name = question_name
+        self.question_text = question_text
+        self.answering_instructions = answering_instructions
+        self.question_presentation = question_presentation
 
-    def translate_answer_code_to_answer(self, answer, scenario: Scenario = None):
-        """Does nothing, because the answer is already in a human-readable format."""
-        return answer
+    @property
+    def question_html_content(self) -> str:
+        from jinja2 import Template
 
-    def simulate_answer(self, human_readable: bool = True) -> dict[str, str]:
-        return {"answer": random_string()}
+        question_html_content = Template(
+            """
+        <div>
+        <textarea id="{{ question_name }}" name="{{ question_name }}"></textarea>
+        </div>
+        """
+        ).render(question_name=self.question_name)
+        return question_html_content
 
     @classmethod
-    def example(cls) -> QuestionFreeText:
-        return cls(
-            question_name="how_are_you",
-            question_text="How are you?",
-            allow_nonresponse=True,
-        )
+    @inject_exception
+    def example(cls, randomize: bool = False) -> QuestionFreeText:
+        """Return an example instance of a free text question."""
+        addition = "" if not randomize else str(uuid4())
+        return cls(question_name="how_are_you", question_text=f"How are you?{addition}")
 
 
 def main():
+    """Create an example question and demonstrate its functionality."""
     from edsl.questions.QuestionFreeText import QuestionFreeText
 
     q = QuestionFreeText.example()
     q.question_text
     q.question_name
-    q.instructions
+    # q.instructions
     # validate an answer
-    q.validate_answer({"answer": "I like custard"})
+    q._validate_answer({"answer": "I like custard"})
     # translate answer code
-    q.translate_answer_code_to_answer({"answer"})
+    q._translate_answer_code_to_answer({"answer"})
     # simulate answer
-    q.simulate_answer()
-    q.simulate_answer(human_readable=False)
-    q.validate_answer(q.simulate_answer(human_readable=False))
+    q._simulate_answer()
+    q._simulate_answer(human_readable=False)
+    q._validate_answer(q._simulate_answer(human_readable=False))
     # serialization (inherits from Question)
     q.to_dict()
     assert q.from_dict(q.to_dict()) == q
+
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/questions/QuestionFunctional.py

@@ -1,69 +1,166 @@
 from typing import Optional, Callable
-from edsl.questions import Question
-from edsl.questions.descriptors import FunctionDescriptor
+import inspect
 
+from edsl.questions.QuestionBase import QuestionBase
+
+from edsl.utilities.restricted_python import create_restricted_function
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
+
+
+class QuestionFunctional(QuestionBase):
+    """A special type of question that is *not* answered by an LLM.
+
+    >>> from edsl import Scenario, Agent
+
+    # Create an instance of QuestionFunctional with the new function
+    >>> question = QuestionFunctional.example()
+
+    # Activate and test the function
+    >>> question.activate()
+    >>> scenario = Scenario({"numbers": [1, 2, 3, 4, 5]})
+    >>> agent = Agent(traits={"multiplier": 10})
+    >>> results = question.by(scenario).by(agent).run(disable_remote_cache = True, disable_remote_inference = True)
+    >>> results.select("answer.*").to_list()[0] == 150
+    True
+
+    # Serialize the question to a dictionary
+
+    >>> from edsl.questions.QuestionBase import QuestionBase
+    >>> new_question = QuestionBase.from_dict(question.to_dict())
+    >>> results = new_question.by(scenario).by(agent).run(disable_remote_cache = True, disable_remote_inference = True)
+    >>> results.select("answer.*").to_list()[0] == 150
+    True
 
-class QuestionFunctional(Question):
-    """
-    A special type of question that's *not* answered by an LLM
-    - Instead, it is "answered" by a function that is passed in, `func`.
-    - Useful for questions that require some kind of computation first
-      or are the result of a multi-step process.
-    See `compose_questions` in `compose_functions.py` for an example of how this is used.
-
-    Notes
-    - 'func' is a function that takes in a scenario and agent traits and returns an answer.
-    - QuestionFunctional is not meant to be instantiated directly by end-users, but rather
-      it is meant to be subclassed by us to create new function types.
-    - It is probably *not* safe to allow end-users to have the ability to pass functional-derived questions.
-      They could monkey-patch the function to do something malicious, e.g., to replace our function logic
-      with "os.system('rm -rf /')".
-    - One possible solution is to have interfaces they can pass via the API, like so:
-      QuestionDropdown(question_name = "dropdown", question_options = ["a", "b", "c"]...)
-      which we then translate to the real QuestionFunctional `under the hood.`
-
-    To see how it's used, see `tests/test_QuestionFunctional_construction_from_function`
     """
 
-    func: Callable = FunctionDescriptor()
     question_type = "functional"
     default_instructions = ""
+    activated = True
+    function_source_code = ""
+    function_name = ""
+
+    _response_model = None
+    response_validator_class = None
 
     def __init__(
         self,
         question_name: str,
-        func: Callable,
+        func: Optional[Callable] = None,
         question_text: Optional[str] = "Functional question",
+        requires_loop: Optional[bool] = False,
+        function_source_code: Optional[str] = None,
+        function_name: Optional[str] = None,
+        unsafe: Optional[bool] = False,
     ):
+        super().__init__()
+        if func:
+            self.function_source_code = inspect.getsource(func)
+            self.function_name = func.__name__
+        else:
+            self.function_source_code = function_source_code
+            self.function_name = function_name
+
+        self.requires_loop = requires_loop
+
+        if unsafe:
+            self.func = func
+        else:
+            self.func = create_restricted_function(
+                self.function_name, self.function_source_code
+            )
+
         self.question_name = question_name
-        self.func = func
         self.question_text = question_text
         self.instructions = self.default_instructions
 
-    def validate_answer(self, answer: dict[str, str]):
-        """Required by Question, but not used by QuestionFunctional"""
-        raise NotImplementedError
+    def activate(self):
+        self.activated = True
+
+    def activate_loop(self):
+        """Activate the function with loop logic using RestrictedPython."""
+        self.func = create_restricted_function(
+            self.function_name, self.function_source_code, loop_activated=True
+        )
 
     def answer_question_directly(self, scenario, agent_traits=None):
-        return {"answer": self.func(scenario, agent_traits), "comment": None}
+        """Return the answer to the question, ensuring the function is activated."""
+        if not self.activated:
+            raise Exception("Function not activated. Please activate it first.")
+        try:
+            return {"answer": self.func(scenario, agent_traits), "comment": None}
+        except Exception as e:
+            print("Function execution error:", e)
+            raise Exception("Error during function execution.")
 
-    def translate_answer_code_to_answer(self, answer, scenario):
-        """Required by Question, but not used by QuestionFunctional"""
+    def _translate_answer_code_to_answer(self, answer, scenario):
+        """Required by Question, but not used by QuestionFunctional."""
         return None
 
-    def simulate_answer(self, human_readable=True) -> dict[str, str]:
-        """Required by Question, but not used by QuestionFunctional"""
+    def _simulate_answer(self, human_readable=True) -> dict[str, str]:
+        """Required by Question, but not used by QuestionFunctional."""
         raise NotImplementedError
 
+    def _validate_answer(self, answer: dict[str, str]):
+        """Required by Question, but not used by QuestionFunctional."""
+        raise NotImplementedError
+
+    @property
+    def question_html_content(self) -> str:
+        return "NA for QuestionFunctional"
+
+    # @add_edsl_version
+    def to_dict(self, add_edsl_version=True):
+        d = {
+            "question_name": self.question_name,
+            "function_source_code": self.function_source_code,
+            "question_type": "functional",
+            "requires_loop": self.requires_loop,
+            "function_name": self.function_name,
+        }
+        if add_edsl_version:
+            from edsl import __version__
+
+            d["edsl_version"] = __version__
+            d["edsl_class_name"] = self.__class__.__name__
+
+        return d
+
     @classmethod
     def example(cls):
-        """Required by Question, but not used by QuestionFunctional"""
-
-        silly_function = lambda scenario, agent_traits: scenario.get(
-            "a", 1
-        ) + scenario.get("b", 2)
         return cls(
-            question_name="add_two_numbers",
-            question_text="functional",
-            func=silly_function,
+            question_name="sum_and_multiply",
+            func=calculate_sum_and_multiply,
+            question_text="Calculate the sum of the list and multiply it by the agent trait multiplier.",
+            requires_loop=True,
         )
+
+
+def calculate_sum_and_multiply(scenario, agent_traits):
+    numbers = scenario.get("numbers", [])
+    multiplier = agent_traits.get("multiplier", 1) if agent_traits else 1
+    sum = 0
+    for num in numbers:
+        sum = sum + num
+    return sum * multiplier
+
+
+def main():
+    from edsl import Scenario, Agent
+    from edsl.questions.QuestionFunctional import QuestionFunctional
+
+    # Create an instance of QuestionFunctional with the new function
+    question = QuestionFunctional.example()
+
+    # Activate and test the function
+    question.activate()
+    scenario = Scenario({"numbers": [1, 2, 3, 4, 5]})
+    agent = Agent(traits={"multiplier": 10})
+    results = question.by(scenario).by(agent).run()
+    assert results.select("answer.*").to_list()[0] == 150
+
+
+if __name__ == "__main__":
+    # main()
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/questions/QuestionList.py

@@ -1,96 +1,223 @@
 from __future__ import annotations
-import random
-import textwrap
+import json
 from typing import Any, Optional, Union
-from edsl.questions import Question
-from edsl.questions.descriptors import (
-    AllowNonresponseDescriptor,
-    IntegerOrNoneDescriptor,
-)
-from edsl.scenarios import Scenario
-from edsl.utilities import random_string
 
+from pydantic import Field
+from json_repair import repair_json
 
-class QuestionList(Question):
-    """
-    This question asks the user to answer by providing a list of items as comma-separated strings.
+from edsl.exceptions.questions import QuestionAnswerValidationError
+from edsl.questions.QuestionBase import QuestionBase
+from edsl.questions.descriptors import IntegerOrNoneDescriptor
+from edsl.questions.decorators import inject_exception
+from edsl.questions.response_validator_abc import ResponseValidatorABC
 
-    Arguments:
-    - `question_name` is the name of the question (string)
-    - `question_text` is the text of the question (string)
 
-    Optional arguments:
-    - `max_list_items` is the maximum number of items that can be in the answer list (positive integer)
-    - `allow_nonresponse` is whether the user can skip the question (boolean). If not provided, the default is False.
-    - `instructions` are the instructions for the question (string). If not provided, the default instructions are used. To view them, run `QuestionList.default_instructions`
+def convert_string(s: str) -> Union[float, int, str, dict]:
+    """Convert a string to a more appropriate type if possible.
 
-    For an example, run `QuestionList.example()`
+    >>> convert_string("3.14")
+    3.14
+    >>> convert_string("42")
+    42
+    >>> convert_string("hello")
+    'hello'
+    >>> convert_string('{"key": "value"}')
+    {'key': 'value'}
+    >>> convert_string("{'key': 'value'}")
+    {'key': 'value'}
     """
 
+    if not isinstance(s, str):  # if it's not a string, return it as is
+        return s
+
+    # If the repair returns, continue on; otherwise, try to load it as JSON
+    if (repaired_json := repair_json(s)) == '""':
+        pass
+    else:
+        try:
+            return json.loads(repaired_json)
+        except json.JSONDecodeError:
+            pass
+
+    # Try to convert to float
+    try:
+        return float(s)
+    except ValueError:
+        pass
+
+    # Try to convert to int
+    try:
+        return int(s)
+    except ValueError:
+        pass
+
+    # If all conversions fail, return the original string
+    return s
+
+
+def create_model(max_list_items: int, permissive: bool) -> "ListResponse":
+    from pydantic import BaseModel
+
+    if permissive or max_list_items is None:
+
+        class ListResponse(BaseModel):
+            answer: list[Any]
+            comment: Optional[str] = None
+            generated_tokens: Optional[str] = None
+
+    else:
+
+        class ListResponse(BaseModel):
+            """
+            >>> nr = ListResponse(answer=["Apple", "Cherry"])
+            >>> nr.dict()
+            {'answer': ['Apple', 'Cherry'], 'comment': None, 'generated_tokens': None}
+            """
+
+            answer: list[Any] = Field(..., min_items=0, max_items=max_list_items)
+            comment: Optional[str] = None
+            generated_tokens: Optional[str] = None
+
+    return ListResponse
+
+
+class ListResponseValidator(ResponseValidatorABC):
+    required_params = ["max_list_items", "permissive"]
+    valid_examples = [({"answer": ["hello", "world"]}, {"max_list_items": 5})]
+
+    invalid_examples = [
+        (
+            {"answer": ["hello", "world", "this", "is", "a", "test"]},
+            {"max_list_items": 5},
+            "Too many items.",
+        ),
+    ]
+
+    def _check_constraints(self, response) -> None:
+        if (
+            self.max_list_items is not None
+            and len(response.answer) > self.max_list_items
+        ):
+            raise QuestionAnswerValidationError("Too many items.")
+
+    def fix(self, response, verbose=False):
+        if verbose:
+            print(f"Fixing list response: {response}")
+        answer = str(response.get("answer") or response.get("generated_tokens", ""))
+        if len(answer.split(",")) > 0:
+            return (
+                {"answer": answer.split(",")} | {"comment": response.get("comment")}
+                if "comment" in response
+                else {}
+            )
+
+    def _post_process(self, edsl_answer_dict):
+        edsl_answer_dict["answer"] = [
+            convert_string(item) for item in edsl_answer_dict["answer"]
+        ]
+        return edsl_answer_dict
+
+
+class QuestionList(QuestionBase):
+    """This question prompts the agent to answer by providing a list of items as comma-separated strings."""
+
     question_type = "list"
     max_list_items: int = IntegerOrNoneDescriptor()
-    allow_nonresponse: bool = AllowNonresponseDescriptor()
+    _response_model = None
+    response_validator_class = ListResponseValidator
 
     def __init__(
         self,
-        question_text: str,
         question_name: str,
-        allow_nonresponse: Optional[bool] = None,
+        question_text: str,
+        include_comment: bool = True,
         max_list_items: Optional[int] = None,
+        answering_instructions: Optional[str] = None,
+        question_presentation: Optional[str] = None,
+        permissive: bool = False,
     ):
-        self.question_text = question_text
+        """Instantiate a new QuestionList.
+
+        :param question_name: The name of the question.
+        :param question_text: The text of the question.
+        :param max_list_items: The maximum number of items that can be in the answer list.
+
+        >>> QuestionList.example().self_check()
+        """
         self.question_name = question_name
-        self.allow_nonresponse = allow_nonresponse or False
+        self.question_text = question_text
         self.max_list_items = max_list_items
+        self.permissive = permissive
+
+        self.include_comment = include_comment
+        self.answering_instructions = answering_instructions
+        self.question_presentations = question_presentation
+
+    def create_response_model(self):
+        return create_model(self.max_list_items, self.permissive)
+
+    @property
+    def question_html_content(self) -> str:
+        from jinja2 import Template
+
+        question_html_content = Template(
+            """
+        <div id="question-list-container">
+            <div>
+                <textarea name="{{ question_name }}[]" rows="1" placeholder="Enter item"></textarea>
+            </div>
+        </div>
+        <button type="button" onclick="addNewLine()">Add another line</button>
+
+        <script>
+            function addNewLine() {
+                var container = document.getElementById('question-list-container');
+                var newLine = document.createElement('div');
+                newLine.innerHTML = '<textarea name="{{ question_name }}[]" rows="1" placeholder="Enter item"></textarea>';
+                container.appendChild(newLine);
+            }
+        </script>
+        """
+        ).render(question_name=self.question_name)
+        return question_html_content
 
-    ################
-    # Answer methods
-    ################
-    def validate_answer(self, answer: Any) -> dict[str, Union[list[str], str]]:
-        """Validates the answer"""
-        self.validate_answer_template_basic(answer)
-        self.validate_answer_key_value(answer, "answer", list)
-        self.validate_answer_list(answer)
-        return answer
-
-    def translate_answer_code_to_answer(self, answer, scenario: Scenario = None):
-        """There is no answer code."""
-        return answer
-
-    def simulate_answer(self, human_readable: bool = True):
-        "Simulates a valid answer for debugging purposes (what the validator expects)"
-        num_items = random.randint(1, self.max_list_items or 2)
-        return {"answer": [random_string() for _ in range(num_items)]}
-
-    ################
-    # Helpful methods
-    ################
     @classmethod
-    def example(cls) -> QuestionList:
+    @inject_exception
+    def example(
+        cls, include_comment=True, max_list_items=None, permissive=False
+    ) -> QuestionList:
+        """Return an example of a list question."""
         return cls(
             question_name="list_of_foods",
             question_text="What are your favorite foods?",
-            allow_nonresponse=False,
-            max_list_items=5,
+            include_comment=include_comment,
+            max_list_items=max_list_items,
+            permissive=permissive,
         )
 
 
 def main():
+    """Create an example of a list question and demonstrate its functionality."""
     from edsl.questions.QuestionList import QuestionList
 
-    q = QuestionList.example()
+    q = QuestionList.example(max_list_items=5)
     q.question_text
     q.question_name
-    q.allow_nonresponse
     q.max_list_items
     # validate an answer
-    q.validate_answer({"answer": ["pasta", "garlic", "oil", "parmesan"]})
+    q._validate_answer({"answer": ["pasta", "garlic", "oil", "parmesan"]})
     # translate answer code
-    q.translate_answer_code_to_answer(["pasta", "garlic", "oil", "parmesan"])
+    q._translate_answer_code_to_answer(["pasta", "garlic", "oil", "parmesan"])
     # simulate answer
-    q.simulate_answer()
-    q.simulate_answer(human_readable=False)
-    q.validate_answer(q.simulate_answer(human_readable=False))
+    q._simulate_answer()
+    q._simulate_answer(human_readable=False)
+    q._validate_answer(q._simulate_answer(human_readable=False))
     # serialization (inherits from Question)
     q.to_dict()
     assert q.from_dict(q.to_dict()) == q
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)