edsl 0.1.33.dev1__py3-none-any.whl → 0.1.33.dev2__py3-none-any.whl

edsl/questions/QuestionCheckBox.py

@@ -10,6 +10,165 @@ from edsl.questions.descriptors import (
     QuestionOptionsDescriptor,
 )
 
+from edsl.questions.decorators import inject_exception
+
+from pydantic import field_validator
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.ResponseValidatorABC import BaseResponse
+
+from edsl.exceptions import QuestionAnswerValidationError
+
+from pydantic import BaseModel, Field, conlist
+from typing import List, Literal, Optional, Annotated
+
+
+def create_checkbox_response_model(
+    choices: list,
+    min_selections: Optional[int] = None,
+    max_selections: Optional[int] = None,
+    permissive: bool = False,
+):
+    """
+    Dynamically create a CheckboxResponse model with a predefined list of choices.
+
+    :param choices: A list of allowed values for the answer field.
+    :param include_comment: Whether to include a comment field in the model.
+    :return: A new Pydantic model class.
+    """
+    # Convert the choices list to a tuple for use with Literal
+    choice_tuple = tuple(choices)
+
+    field_params = {}
+    if min_selections is not None and not permissive:
+        field_params["min_items"] = min_selections
+    if max_selections is not None and not permissive:
+        field_params["max_items"] = max_selections
+
+    class CheckboxResponse(BaseModel):
+        answer: Annotated[
+            List[Literal[choice_tuple]],
+            Field(..., **field_params),
+        ] = Field(..., description="List of selected choices")
+        comment: Optional[str] = Field(None, description="Optional comment field")
+        generated_tokens: Optional[Any] = Field(default=None)
+
+        class Config:
+            @staticmethod
+            def json_schema_extra(schema: dict, model: BaseModel) -> None:
+                # Add the list of choices to the schema for better documentation
+                for prop in schema.get("properties", {}).values():
+                    if prop.get("title") == "answer":
+                        prop["items"] = {"enum": choices}
+
+    return CheckboxResponse
+
+
+class CheckBoxResponseValidator(ResponseValidatorABC):
+    required_params = [
+        "question_options",
+        "min_selections",
+        "max_selections",
+        "use_code",
+        "permissive",
+    ]
+
+    valid_examples = [
+        ({"answer": [1, 2]}, {"question_options": ["Good", "Great", "OK", "Bad"]})
+    ]
+
+    invalid_examples = [
+        (
+            {"answer": [-1]},
+            {"question_options": ["Good", "Great", "OK", "Bad"]},
+            "Answer code must be a non-negative integer",
+        ),
+        (
+            {"answer": 1},
+            {"question_options": ["Good", "Great", "OK", "Bad"]},
+            "Answer code must be a list",
+        ),
+        (
+            {"answer": [1, 2, 3, 4]},
+            {
+                "question_options": ["Good", "Great", "OK", "Bad"],
+                "min_selections": 1,
+                "max_selections": 2,
+            },
+            "Too many options selected",
+        ),
+    ]
+
+    def fix(self, response, verbose=False):
+        if verbose:
+            print("Invalid response of QuestionCheckBox was: ", response)
+        response_text = response.get("generated_tokens")
+        if response_text is None or response_text == "":  # nothing to be done
+            return response
+        # Maybe it's a comma separated list?
+        proposed_list = response_text.split(",")
+        proposed_list = [item.strip() for item in proposed_list]
+        if verbose:
+            print("Using code? ", self.use_code)
+        if self.use_code:
+            try:
+                proposed_list = [int(i) for i in proposed_list]
+            except ValueError:
+                # print("Could not convert to int")
+                pass
+
+        if verbose:
+            print("Proposed solution is: ", proposed_list)
+
+        # print(f"Ivalid generated tokens was was: {response_text}")
+        if "comment" in response:
+            proposed_data = {
+                "answer": proposed_list,
+                "comment": response["comment"],
+                "generated_tokens": response.get("generated_tokens", None),
+            }
+        else:
+            proposed_data = {
+                "answer": proposed_list,
+                "generated_tokens": response.get("generated_tokens", None),
+            }
+
+        try:
+            self.response_model(**proposed_data)
+            print("Proposed solution is valid")
+            print("Returning proposed data: ", proposed_data)
+            return proposed_data
+        except Exception as e:
+            if verbose:
+                print(f"Proposed solution {proposed_data} is invalid. Error: {e}")
+            # return response
+        if verbose:
+            print("Now seeing if responses show up in the answer")
+        matches = []
+        for index, option in enumerate(self.question_options):
+            if self.use_code:
+                if str(index) in response_text:
+                    matches.append(index)
+            else:
+                if option in response_text:
+                    matches.append(index)
+        proposed_data = {
+            "answer": matches,
+            "comment": response.get("comment", None),
+            "generated_tokens": response.get("generated_tokens", None),
+        }
+        try:
+            self.response_model(**proposed_data)
+            return proposed_data
+        except Exception as e:
+            if verbose:
+                print(f"Proposed solution {proposed_data} is invalid. Error: {e}")
+            return response
+
+    def custom_validate(self, response) -> BaseResponse:
+        if response.answer is None:
+            raise QuestionAnswerValidationError("Answer is missing.")
+        return response.dict()
+
 
 class QuestionCheckBox(QuestionBase):
     """This question prompts the agent to select options from a list."""
@@ -20,6 +179,9 @@ class QuestionCheckBox(QuestionBase):
     min_selections = IntegerDescriptor(none_allowed=True)
     max_selections = IntegerDescriptor(none_allowed=True)
 
+    _response_model = None
+    response_validator_class = CheckBoxResponseValidator
+
     def __init__(
         self,
         question_name: str,
@@ -27,6 +189,11 @@ class QuestionCheckBox(QuestionBase):
         question_options: list[str],
         min_selections: Optional[int] = None,
         max_selections: Optional[int] = None,
+        include_comment: bool = True,
+        use_code: bool = True,
+        question_presentation: Optional[str] = None,
+        answering_instructions: Optional[str] = None,
+        permissive: bool = False,
     ):
         """Instantiate a new QuestionCheckBox.
 
@@ -42,15 +209,28 @@ class QuestionCheckBox(QuestionBase):
         self.max_selections = max_selections
         self.question_options = question_options
 
-    ################
-    # Answer methods
-    ################
-    def _validate_answer(self, answer: Any) -> dict[str, Union[int, str]]:
-        """Validate the answer."""
-        self._validate_answer_template_basic(answer)
-        self._validate_answer_key_value(answer, "answer", list)
-        self._validate_answer_checkbox(answer)
-        return answer
+        self._include_comment = include_comment
+        self._use_code = use_code
+        self.permissive = permissive
+
+        self.question_presentation = question_presentation
+        self.answering_instructions = answering_instructions
+
+    def create_response_model(self):
+        if not self._use_code:
+            return create_checkbox_response_model(
+                self.question_options,
+                min_selections=self.min_selections,
+                max_selections=self.max_selections,  # include_comment=self._include_comment
+                permissive=self.permissive,
+            )
+        else:
+            return create_checkbox_response_model(
+                list(range(len(self.question_options))),
+                min_selections=self.min_selections,
+                max_selections=self.max_selections,  # include_comment=self._include_comment
+                permissive=self.permissive,
+            )
 
     def _translate_answer_code_to_answer(
         self, answer_codes, scenario: "Scenario" = None
@@ -69,33 +249,36 @@ class QuestionCheckBox(QuestionBase):
         ]
         translated_codes = []
         for answer_code in answer_codes:
-            translated_codes.append(translated_options[int(answer_code)])
+            if self._use_code:
+                translated_codes.append(translated_options[int(answer_code)])
+            else:
+                translated_codes.append(answer_code)
         return translated_codes
 
-    def _simulate_answer(self, human_readable=True) -> dict[str, Union[int, str]]:
-        """Simulate a valid answer for debugging purposes."""
-        from edsl.utilities.utilities import random_string
-
-        min_selections = self.min_selections or 1
-        max_selections = self.max_selections or len(self.question_options)
-        num_selections = random.randint(min_selections, max_selections)
-        if human_readable:
-            # Select a random number of options from self.question_options
-            selected_options = random.sample(self.question_options, num_selections)
-            answer = {
-                "answer": selected_options,
-                "comment": random_string(),
-            }
-        else:
-            # Select a random number of indices from the range of self.question_options
-            selected_indices = random.sample(
-                range(len(self.question_options)), num_selections
-            )
-            answer = {
-                "answer": selected_indices,
-                "comment": random_string(),
-            }
-        return answer
+    # def _simulate_answer(self, human_readable=True) -> dict[str, Union[int, str]]:
+    #     """Simulate a valid answer for debugging purposes."""
+    #     from edsl.utilities.utilities import random_string
+
+    #     min_selections = self.min_selections or 1
+    #     max_selections = self.max_selections or len(self.question_options)
+    #     num_selections = random.randint(min_selections, max_selections)
+    #     if human_readable:
+    #         # Select a random number of options from self.question_options
+    #         selected_options = random.sample(self.question_options, num_selections)
+    #         answer = {
+    #             "answer": selected_options,
+    #             "comment": random_string(),
+    #         }
+    #     else:
+    #         # Select a random number of indices from the range of self.question_options
+    #         selected_indices = random.sample(
+    #             range(len(self.question_options)), num_selections
+    #         )
+    #         answer = {
+    #             "answer": selected_indices,
+    #             "comment": random_string(),
+    #         }
+    #     return answer
 
     @property
     def question_html_content(self) -> str:
@@ -125,7 +308,8 @@ class QuestionCheckBox(QuestionBase):
     # Helpful methods
     ################
     @classmethod
-    def example(cls) -> QuestionCheckBox:
+    @inject_exception
+    def example(cls, include_comment=False, use_code=True) -> QuestionCheckBox:
         """Return an example checkbox question."""
         return cls(
             question_name="never_eat",
@@ -139,6 +323,8 @@ class QuestionCheckBox(QuestionBase):
             ],
             min_selections=2,
             max_selections=5,
+            use_code=use_code,
+            include_comment=include_comment,
         )
 
 
@@ -165,3 +351,9 @@ def main():
     import doctest
 
     doctest.testmod(optionflags=doctest.ELLIPSIS)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/questions/QuestionExtract.py

@@ -1,20 +1,112 @@
 from __future__ import annotations
-from typing import Any
+from typing import Any, Optional, Dict
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import AnswerTemplateDescriptor
 
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.ResponseValidatorABC import BaseResponse
+from edsl.exceptions import QuestionAnswerValidationError
+from edsl.questions.decorators import inject_exception
+
+from typing import Dict, Any
+from pydantic import create_model, Field
+
+import json
+import re
+
+
+def extract_json(text, expected_keys, verbose=False):
+    # Escape special regex characters in keys
+    escaped_keys = [re.escape(key) for key in expected_keys]
+
+    # Create a pattern that looks for all expected keys
+    pattern = r"\{[^}]*" + r"[^}]*".join(escaped_keys) + r"[^}]*\}"
+
+    json_match = re.search(pattern, text)
+
+    if json_match:
+        json_str = json_match.group(0)
+        try:
+            # Parse the extracted string as JSON
+            json_data = json.loads(json_str)
+
+            # Verify that all expected keys are present
+            if all(key in json_data for key in expected_keys):
+                return json_data
+            else:
+                if verbose:
+                    print(
+                        "Error: Not all expected keys were found in the extracted JSON."
+                    )
+                return None
+        except json.JSONDecodeError:
+            if verbose:
+                print("Error: The extracted content is not valid JSON.")
+            return None
+    else:
+        if verbose:
+            print("Error: No JSON-like structure found with all expected keys.")
+        return None
+
+
+def dict_to_pydantic_model(input_dict: Dict[str, Any]) -> Any:
+    field_definitions = {
+        key: (str, Field(default=str(value))) for key, value in input_dict.items()
+    }
+
+    DynamicModel = create_model("DynamicModel", **field_definitions)
+
+    class AnswerModel(BaseResponse):
+        answer: DynamicModel
+        generated_tokens: Optional[str] = None
+        comment: Optional[str] = None
+
+    return AnswerModel
+
+
+class ExtractResponseValidator(ResponseValidatorABC):
+    required_params = ["answer_template"]
+    valid_examples = [({"answer": "This is great"}, {})]
+    invalid_examples = [
+        (
+            {"answer": None},
+            {"answer_template": {"name": "John Doe", "profession": "Carpenter"}},
+            "Result cannot be empty",
+        ),
+    ]
+
+    def custom_validate(self, response) -> BaseResponse:
+        return response.dict()
+
+    def fix(self, response, verbose=False):
+        raw_tokens = response["generated_tokens"]
+        if verbose:
+            print(f"Invalid response of QuestionExtract was: {raw_tokens}")
+        extracted_json = extract_json(raw_tokens, self.answer_template.keys(), verbose)
+        if verbose:
+            print("Proposed solution is: ", extracted_json)
+        return {
+            "answer": extracted_json,
+            "comment": response.get("comment", None),
+            "generated_tokens": raw_tokens,
+        }
+
 
 class QuestionExtract(QuestionBase):
     """This question prompts the agent to extract information from a string and return it in a given template."""
 
     question_type = "extract"
     answer_template: dict[str, Any] = AnswerTemplateDescriptor()
+    _response_model = None
+    response_validator_class = ExtractResponseValidator
 
     def __init__(
         self,
         question_text: str,
         answer_template: dict[str, Any],
         question_name: str,
+        answering_instructions: str = None,
+        question_presentation: str = None,
     ):
         """Initialize the question.
 
@@ -26,33 +118,11 @@ class QuestionExtract(QuestionBase):
         self.question_name = question_name
         self.question_text = question_text
         self.answer_template = answer_template
+        self.answering_instructions = answering_instructions
+        self.question_presentation = question_presentation
 
-    ################
-    # Answer methods
-    ################
-    def _validate_answer(self, answer: Any) -> dict[str, Any]:
-        """Validate the answer."""
-        # raw_json = answer["answer"]
-        # fixed_json_data = re.sub(r"\'", '"', raw_json)
-        # answer["answer"] = json.loads(fixed_json_data)
-        self._validate_answer_template_basic(answer)
-        # self._validate_answer_key_value(answer, "answer", dict)
-
-        self._validate_answer_extract(answer)
-        return answer
-
-    def _translate_answer_code_to_answer(self, answer, scenario: "Scenario" = None):
-        """Return the answer in a human-readable format."""
-        return answer
-
-    def _simulate_answer(self, human_readable: bool = True) -> dict[str, str]:
-        """Simulate a valid answer for debugging purposes."""
-        from edsl.utilities.utilities import random_string
-
-        return {
-            "answer": {key: random_string() for key in self.answer_template.keys()},
-            "comment": random_string(),
-        }
+    def create_response_model(self):
+        return dict_to_pydantic_model(self.answer_template)
 
     @property
     def question_html_content(self) -> str:
@@ -77,6 +147,7 @@ class QuestionExtract(QuestionBase):
     # Helpful methods
     ################
     @classmethod
+    @inject_exception
     def example(cls) -> QuestionExtract:
         """Return an example question."""
         return cls(

edsl/questions/QuestionFreeText.py

@@ -1,23 +1,56 @@
 from __future__ import annotations
-import textwrap
 from typing import Any, Optional
 from uuid import uuid4
+
+from pydantic import field_validator
+
 from edsl.questions.QuestionBase import QuestionBase
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+
+from edsl.exceptions import QuestionAnswerValidationError
+from edsl.questions.decorators import inject_exception
+
+from pydantic import BaseModel
+from typing import Optional, Any, List
+
+from edsl.exceptions import QuestionAnswerValidationError
+
+
+class FreeTextResponse(BaseModel):
+    """
+    Validator for free text response questions.
+    """
+
+    answer: str
+    generated_tokens: Optional[str] = None
+
+
+class FreeTextResponseValidator(ResponseValidatorABC):
+    required_params = []
+    valid_examples = [({"answer": "This is great"}, {})]
+    invalid_examples = [
+        (
+            {"answer": None},
+            {},
+            "Answer code must not be missing.",
+        ),
+    ]
 
 
 class QuestionFreeText(QuestionBase):
     """This question prompts the agent to respond with free text."""
 
     question_type = "free_text"
-    default_instructions = textwrap.dedent(
-        """\
-        You are being asked the following question: {{question_text}}
-        Return a valid JSON formatted like this: 
-        {"answer": "<put free text answer here>"}
-        """
-    )
-
-    def __init__(self, question_name: str, question_text: str):
+    _response_model = FreeTextResponse
+    response_validator_class = FreeTextResponseValidator
+
+    def __init__(
+        self,
+        question_name: str,
+        question_text: str,
+        answering_instructions: Optional[str] = None,
+        question_presentation: Optional[str] = None,
+    ):
         """Instantiate a new QuestionFreeText.
 
         :param question_name: The name of the question.
@@ -25,25 +58,8 @@ class QuestionFreeText(QuestionBase):
         """
         self.question_name = question_name
         self.question_text = question_text
-
-    ################
-    # Answer methods
-    ################
-    def _validate_answer(self, answer: Any) -> dict[str, str]:
-        """Validate the answer."""
-        self._validate_answer_template_basic(answer)
-        self._validate_answer_key_value(answer, "answer", str)
-        return answer
-
-    def _translate_answer_code_to_answer(self, answer, scenario: "Scenario" = None):
-        """Do nothing, because the answer is already in a human-readable format."""
-        return answer
-
-    def _simulate_answer(self, human_readable: bool = True) -> dict[str, str]:
-        """Simulate a valid answer for debugging purposes."""
-        from edsl.utilities.utilities import random_string
-
-        return {"answer": random_string()}
+        self.answering_instructions = answering_instructions
+        self.question_presentation = question_presentation
 
     @property
     def question_html_content(self) -> str:
@@ -59,6 +75,7 @@ class QuestionFreeText(QuestionBase):
         return question_html_content
 
     @classmethod
+    @inject_exception
     def example(cls, randomize: bool = False) -> QuestionFreeText:
         """Return an example instance of a free text question."""
         addition = "" if not randomize else str(uuid4())

edsl/questions/QuestionFunctional.py

@@ -39,6 +39,9 @@ class QuestionFunctional(QuestionBase):
     function_source_code = ""
     function_name = ""
 
+    _response_model = None
+    response_validator_class = None
+
     def __init__(
         self,
         question_name: str,
@@ -97,6 +100,10 @@ class QuestionFunctional(QuestionBase):
         """Required by Question, but not used by QuestionFunctional."""
         raise NotImplementedError
 
+    @property
+    def question_html_content(self) -> str:
+        return "NA for QuestionFunctional"
+
     @add_edsl_version
     def to_dict(self):
         return {