edsl 0.1.31.dev4__py3-none-any.whl → 0.1.33__py3-none-any.whl

edsl/questions/QuestionList.py

@@ -4,6 +4,123 @@ import textwrap
 from typing import Any, Optional, Union
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import IntegerOrNoneDescriptor
+from edsl.questions.decorators import inject_exception
+
+from pydantic import field_validator, Field
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.ResponseValidatorABC import BaseResponse
+
+from edsl.exceptions import QuestionAnswerValidationError
+import textwrap
+import json
+
+from json_repair import repair_json
+
+
+def convert_string(s):
+    """Convert a string to a more appropriate type if possible.
+
+    >>> convert_string("3.14")
+    3.14
+    >>> convert_string("42")
+    42
+    >>> convert_string("hello")
+    'hello'
+    >>> convert_string('{"key": "value"}')
+    {'key': 'value'}
+    >>> convert_string("{'key': 'value'}")
+    {'key': 'value'}
+    """
+
+    if not isinstance(s, str):  # if it's not a string, return it as is
+        return s
+
+    # If the repair returns, continue on; otherwise, try to load it as JSON
+    if (repaired_json := repair_json(s)) == '""':
+        pass
+    else:
+        try:
+            return json.loads(repaired_json)
+        except json.JSONDecodeError:
+            pass
+
+    # Try to convert to float
+    try:
+        return float(s)
+    except ValueError:
+        pass
+
+    # Try to convert to int
+    try:
+        return int(s)
+    except ValueError:
+        pass
+
+    # If all conversions fail, return the original string
+    return s
+
+
+def create_model(max_list_items: int, permissive):
+    from pydantic import BaseModel
+
+    if permissive or max_list_items is None:
+
+        class ListResponse(BaseModel):
+            answer: list[Any]
+            comment: Optional[str] = None
+            generated_tokens: Optional[str] = None
+
+    else:
+
+        class ListResponse(BaseModel):
+            """
+            >>> nr = ListResponse(answer=["Apple", "Cherry"])
+            >>> nr.dict()
+            {'answer': ['Apple', 'Cherry'], 'comment': None, 'generated_tokens': None}
+            """
+
+            answer: list[Any] = Field(..., min_items=0, max_items=max_list_items)
+            comment: Optional[str] = None
+            generated_tokens: Optional[str] = None
+
+    return ListResponse
+
+
+class ListResponseValidator(ResponseValidatorABC):
+    required_params = ["max_list_items", "permissive"]
+    valid_examples = [({"answer": ["hello", "world"]}, {"max_list_items": 5})]
+
+    invalid_examples = [
+        (
+            {"answer": ["hello", "world", "this", "is", "a", "test"]},
+            {"max_list_items": 5},
+            "Too many items.",
+        ),
+    ]
+
+    def _check_constraints(self, response) -> None:
+        if (
+            self.max_list_items is not None
+            and len(response.answer) > self.max_list_items
+        ):
+            raise QuestionAnswerValidationError("Too many items.")
+
+    def fix(self, response, verbose=False):
+        if verbose:
+            print(f"Fixing list response: {response}")
+        answer = str(response.get("answer") or response.get("generated_tokens", ""))
+        if len(answer.split(",")) > 0:
+            return (
+                {"answer": answer.split(",")} | {"comment": response.get("comment")}
+                if "comment" in response
+                else {}
+            )
+
+    def _post_process(self, edsl_answer_dict):
+        edsl_answer_dict["answer"] = [
+            convert_string(item) for item in edsl_answer_dict["answer"]
+        ]
+        return edsl_answer_dict
 
 
 class QuestionList(QuestionBase):
@@ -11,44 +128,38 @@ class QuestionList(QuestionBase):
 
     question_type = "list"
     max_list_items: int = IntegerOrNoneDescriptor()
+    _response_model = None
+    response_validator_class = ListResponseValidator
 
     def __init__(
         self,
         question_name: str,
         question_text: str,
         max_list_items: Optional[int] = None,
+        include_comment: bool = True,
+        answering_instructions: Optional[str] = None,
+        question_presentation: Optional[str] = None,
+        permissive: bool = False,
     ):
         """Instantiate a new QuestionList.
 
         :param question_name: The name of the question.
         :param question_text: The text of the question.
-        :param instructions: Instructions for the question. If not provided, the default instructions are used. To view them, run `QuestionList.default_instructions`.
         :param max_list_items: The maximum number of items that can be in the answer list.
+
+        >>> QuestionList.example().self_check()
         """
         self.question_name = question_name
         self.question_text = question_text
         self.max_list_items = max_list_items
+        self.permissive = permissive
 
-    ################
-    # Answer methods
-    ################
-    def _validate_answer(self, answer: Any) -> dict[str, Union[list[str], str]]:
-        """Validate the answer."""
-        self._validate_answer_template_basic(answer)
-        self._validate_answer_key_value(answer, "answer", list)
-        self._validate_answer_list(answer)
-        return answer
-
-    def _translate_answer_code_to_answer(self, answer, scenario: "Scenario" = None):
-        """There is no answer code."""
-        return answer
-
-    def _simulate_answer(self, human_readable: bool = True):
-        """Simulate a valid answer for debugging purposes (what the validator expects)."""
-        num_items = random.randint(1, self.max_list_items or 2)
-        from edsl.utilities.utilities import random_string
+        self.include_comment = include_comment
+        self.answering_instructions = answering_instructions
+        self.question_presentations = question_presentation
 
-        return {"answer": [random_string() for _ in range(num_items)]}
+    def create_response_model(self):
+        return create_model(self.max_list_items, self.permissive)
 
     @property
     def question_html_content(self) -> str:
@@ -79,12 +190,17 @@ class QuestionList(QuestionBase):
     # Helpful methods
     ################
     @classmethod
-    def example(cls) -> QuestionList:
+    @inject_exception
+    def example(
+        cls, include_comment=True, max_list_items=None, permissive=False
+    ) -> QuestionList:
         """Return an example of a list question."""
         return cls(
             question_name="list_of_foods",
             question_text="What are your favorite foods?",
-            max_list_items=5,
+            include_comment=include_comment,
+            max_list_items=max_list_items,
+            permissive=permissive,
         )
 
 
@@ -92,7 +208,7 @@ def main():
     """Create an example of a list question and demonstrate its functionality."""
     from edsl.questions.QuestionList import QuestionList
 
-    q = QuestionList.example()
+    q = QuestionList.example(max_list_items=5)
     q.question_text
     q.question_name
     q.max_list_items
@@ -108,6 +224,8 @@ def main():
     q.to_dict()
     assert q.from_dict(q.to_dict()) == q
 
+
+if __name__ == "__main__":
     import doctest
 
     doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/questions/QuestionMultipleChoice.py

@@ -1,12 +1,114 @@
 from __future__ import annotations
-import time
-from typing import Union
-import random
-from typing import Optional
+from typing import Union, Literal, Optional, List, Any
+
 from jinja2 import Template
+from pydantic import BaseModel, Field
 
+from edsl.scenarios.Scenario import Scenario
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import QuestionOptionsDescriptor
+from edsl.questions.decorators import inject_exception
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+
+
+def create_response_model(choices: List[str], permissive: bool = False):
+    """
+    Create a ChoiceResponse model class with a predefined list of choices.
+
+    :param choices: A list of allowed values for the answer field.
+    :param permissive: If True, any value will be accepted as an answer.
+    :return: A new Pydantic model class.
+    """
+    choice_tuple = tuple(choices)
+
+    if not permissive:
+
+        class ChoiceResponse(BaseModel):
+            answer: Literal[choice_tuple] = Field(description="Selected choice")
+            comment: Optional[str] = Field(None, description="Optional comment field")
+            generated_tokens: Optional[Any] = Field(
+                None, description="Generated tokens"
+            )
+
+            class Config:
+                @staticmethod
+                def json_schema_extra(schema: dict, model: BaseModel) -> None:
+                    for prop in schema.get("properties", {}).values():
+                        if prop.get("title") == "answer":
+                            prop["enum"] = choices
+
+    else:
+
+        class ChoiceResponse(BaseModel):
+            answer: Any = Field(description="Selected choice (can be any value)")
+            comment: Optional[str] = Field(None, description="Optional comment field")
+            generated_tokens: Optional[Any] = Field(
+                None, description="Generated tokens"
+            )
+
+            class Config:
+                @staticmethod
+                def json_schema_extra(schema: dict, model: BaseModel) -> None:
+                    for prop in schema.get("properties", {}).values():
+                        if prop.get("title") == "answer":
+                            prop["description"] += f". Suggested choices are: {choices}"
+                    schema["title"] += " (Permissive)"
+
+    return ChoiceResponse
+
+
+class MultipleChoiceResponseValidator(ResponseValidatorABC):
+    required_params = ["question_options", "use_code"]
+
+    def fix(self, response, verbose=False):
+        response_text = str(response.get("answer"))
+        if response_text is None:
+            response_text = response.get("generated_tokens", "")
+
+        if verbose:
+            print(f"Invalid generated tokens was: {response_text}")
+
+        matches = []
+        for idx, option in enumerate(self.question_options):
+            if verbose:
+                print("The options are: ", self.question_options)
+            if str(option) in response_text:
+                if verbose:
+                    print("Match found with option ", option)
+                if option not in matches:
+                    matches.append(option)
+
+        if verbose:
+            print("The matches are: ", matches)
+        if len(matches) == 1:
+            proposed_data = {
+                "answer": matches[0],
+                "generated_tokens": response.get("generated_tokens", None),
+            }
+            try:
+                self.response_model(**proposed_data)
+                return proposed_data
+            except Exception as e:
+                if verbose:
+                    print(f"Proposed solution {proposed_data} is invalid. Error: {e}")
+            return response
+
+    valid_examples = [
+        ({"answer": 1}, {"question_options": ["Good", "Great", "OK", "Bad"]})
+    ]
+
+    invalid_examples = [
+        (
+            {"answer": -1},
+            {"question_options": ["Good", "Great", "OK", "Bad"]},
+            "Answer code must be a non-negative integer",
+        ),
+        (
+            {"answer": None},
+            {"question_options": ["Good", "Great", "OK", "Bad"]},
+            "Answer code must not be missing.",
+        ),
+    ]
 
 
 class QuestionMultipleChoice(QuestionBase):
@@ -21,50 +123,53 @@ class QuestionMultipleChoice(QuestionBase):
     question_options: Union[
         list[str], list[list], list[float], list[int]
     ] = QuestionOptionsDescriptor()
+    _response_model = None
+    response_validator_class = MultipleChoiceResponseValidator
 
     def __init__(
         self,
         question_name: str,
         question_text: str,
         question_options: Union[list[str], list[list], list[float], list[int]],
+        include_comment: bool = True,
+        use_code: bool = False,
+        answering_instructions: Optional[str] = None,
+        question_presentation: Optional[str] = None,
+        permissive: bool = False,
     ):
         """Instantiate a new QuestionMultipleChoice.
 
         :param question_name: The name of the question.
         :param question_text: The text of the question.
         :param question_options: The options the agent should select from.
-        :param instructions: Instructions for the question. If not provided, the default instructions are used. To view them, run `QuestionMultipleChoice.default_instructions`.
+        :param include_comment: Whether to include a comment field.
+        :param use_code: Whether to use code for the options.
+        :param answering_instructions: Instructions for the question.
+        :param question_presentation: The presentation of the question.
+        :param permissive: Whether to force the answer to be one of the options.
+
         """
         self.question_name = question_name
         self.question_text = question_text
         self.question_options = question_options
 
-    # @property
-    # def question_options(self) -> Union[list[str], list[list], list[float], list[int]]:
-    #     """Return the question options."""
-    #     return self._question_options
+        self._include_comment = include_comment
+        self.use_code = use_code
+        self.answering_instructions = answering_instructions
+        self.question_presentation = question_presentation
+        self.permissive = permissive
 
     ################
     # Answer methods
     ################
-    def _validate_answer(
-        self, answer: dict[str, Union[str, int]]
-    ) -> dict[str, Union[str, int]]:
-        """Validate the answer.
 
-        >>> q = QuestionMultipleChoice.example()
-        >>> q._validate_answer({"answer": 0, "comment": "I like custard"})
-        {'answer': 0, 'comment': 'I like custard'}
-
-        >>> q = QuestionMultipleChoice(question_name="how_feeling", question_text="How are you?", question_options=["Good", "Great", "OK", "Bad"])
-        >>> q._validate_answer({"answer": -1, "comment": "I like custard"})
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionAnswerValidationError: Answer code must be a non-negative integer (got -1).
-        """
-        self._validate_answer_template_basic(answer)
-        self._validate_answer_multiple_choice(answer)
-        return answer
+    def create_response_model(self):
+        if self.use_code:
+            return create_response_model(
+                list(range(len(self.question_options))), self.permissive
+            )
+        else:
+            return create_response_model(self.question_options, self.permissive)
 
     def _translate_answer_code_to_answer(
         self, answer_code: int, scenario: Optional["Scenario"] = None
@@ -75,15 +180,14 @@ class QuestionMultipleChoice(QuestionBase):
         The question options might be templates, so they need to be rendered with the scenario.
 
         >>> q = QuestionMultipleChoice.example()
-        >>> q._translate_answer_code_to_answer(0, {})
+        >>> q._translate_answer_code_to_answer('Good', {})
         'Good'
 
         >>> q = QuestionMultipleChoice(question_name="how_feeling", question_text="How are you?", question_options=["{{emotion[0]}}", "emotion[1]"])
-        >>> q._translate_answer_code_to_answer(0, {"emotion": ["Happy", "Sad"]})
+        >>> q._translate_answer_code_to_answer('Happy', {"emotion": ["Happy", "Sad"]})
         'Happy'
 
         """
-        from edsl.scenarios.Scenario import Scenario
 
         scenario = scenario or Scenario()
 
@@ -96,31 +200,17 @@ class QuestionMultipleChoice(QuestionBase):
             question_option_key = list(meta.find_undeclared_variables(parsed_content))[
                 0
             ]
-            #breakpoint()
             translated_options = scenario.get(question_option_key)
         else:
             translated_options = [
                 Template(str(option)).render(scenario)
                 for option in self.question_options
             ]
-        # print("Translated options:", translated_options)
-        # breakpoint()
-        return translated_options[int(answer_code)]
-
-    def _simulate_answer(
-        self, human_readable: bool = True
-    ) -> dict[str, Union[int, str]]:
-        """Simulate a valid answer for debugging purposes."""
-        from edsl.utilities.utilities import random_string
-
-        if human_readable:
-            answer = random.choice(self.question_options)
+        if self._use_code:
+            return translated_options[int(answer_code)]
         else:
-            answer = random.choice(range(len(self.question_options)))
-        return {
-            "answer": answer,
-            "comment": random_string(),
-        }
+            # return translated_options[answer_code]
+            return answer_code
 
     @property
     def question_html_content(self) -> str:
@@ -154,33 +244,36 @@ class QuestionMultipleChoice(QuestionBase):
     # Example
     ################
     @classmethod
-    def example(cls) -> QuestionMultipleChoice:
+    @inject_exception
+    def example(cls, include_comment=False, use_code=False) -> QuestionMultipleChoice:
         """Return an example instance."""
         return cls(
             question_text="How are you?",
             question_options=["Good", "Great", "OK", "Bad"],
             question_name="how_feeling",
+            include_comment=include_comment,
+            use_code=use_code,
         )
 
 
-def main():
-    """Create an example QuestionMultipleChoice and test its methods."""
-    from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
-
-    q = QuestionMultipleChoice.example()
-    q.question_text
-    q.question_options
-    q.question_name
-    # validate an answer
-    q._validate_answer({"answer": 0, "comment": "I like custard"})
-    # translate answer code
-    q._translate_answer_code_to_answer(0, {})
-    # simulate answer
-    q._simulate_answer()
-    q._simulate_answer(human_readable=False)
-    # serialization (inherits from Question)
-    q.to_dict()
-    assert q.from_dict(q.to_dict()) == q
+# def main():
+#     """Create an example QuestionMultipleChoice and test its methods."""
+#     from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
+
+#     q = QuestionMultipleChoice.example()
+#     q.question_text
+#     q.question_options
+#     q.question_name
+#     # validate an answer
+#     q._validate_answer({"answer": 0, "comment": "I like custard"})
+#     # translate answer code
+#     q._translate_answer_code_to_answer(0, {})
+#     # simulate answer
+#     q._simulate_answer()
+#     q._simulate_answer(human_readable=False)
+#     # serialization (inherits from Question)
+#     q.to_dict()
+#     assert q.from_dict(q.to_dict()) == q
 
 
 if __name__ == "__main__":