edsl 0.1.33.dev1__py3-none-any.whl → 0.1.33.dev2__py3-none-any.whl

edsl/questions/QuestionNumerical.py

@@ -1,26 +1,97 @@
 from __future__ import annotations
-import textwrap
+from decimal import Decimal
 from random import uniform
-from typing import Any, Optional, Union
+from typing import Any, Optional, Union, Literal
+
+from pydantic import BaseModel, Field, field_validator
 
 from edsl.exceptions import QuestionAnswerValidationError
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import NumericalOrNoneDescriptor
+from edsl.questions.decorators import inject_exception
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.exceptions.questions import QuestionAnswerValidationError
+
+
+def create_numeric_response(
+    min_value: Optional[Decimal] = None,
+    max_value: Optional[Decimal] = None,
+    permissive=False,
+):
+    field_kwargs = {}
+    if not permissive:
+        field_kwargs = {}
+        if min_value is not None:
+            field_kwargs["ge"] = min_value
+        if max_value is not None:
+            field_kwargs["le"] = max_value
+
+    class ConstrainedNumericResponse(BaseModel):
+        answer: Union[Decimal] = Field(**field_kwargs)
+        comment: Optional[str] = Field(None)
+        generated_tokens: Optional[Any] = Field(None)
+
+    return ConstrainedNumericResponse
+
+
+class NumericalResponseValidator(ResponseValidatorABC):
+    required_params = ["min_value", "max_value", "permissive"]
+
+    valid_examples = [
+        ({"answer": 1}, {"min_value": 0, "max_value": 10}),
+        ({"answer": 1}, {"min_value": None, "max_value": None}),
+    ]
+
+    invalid_examples = [
+        ({"answer": 10}, {"min_value": 0, "max_value": 5}, "Answer is out of range"),
+        ({"answer": "ten"}, {"min_value": 0, "max_value": 5}, "Answer is not a number"),
+        ({}, {"min_value": 0, "max_value": 5}, "Answer key is missing"),
+    ]
+
+    def fix(self, response, verbose=False):
+        response_text = str(response).lower()
+        import re
+
+        if verbose:
+            print(f"Ivalid generated tokens was was: {response_text}")
+        pattern = r"\b\d+(?:\.\d+)?\b"
+        match = re.search(pattern, response_text.replace(",", ""))
+        solution = match.group(0) if match else response.get("answer")
+        if verbose:
+            print("Proposed solution is: ", solution)
+        if "comment" in response:
+            return {"answer": solution, "comment": response["comment"]}
+        else:
+            return {"answer": solution}
+
+    def _check_constraints(self, pydantic_edsl_answer: BaseModel):
+        pass
 
 
 class QuestionNumerical(QuestionBase):
-    """This question prompts the agent to answer with a numerical value."""
+    """This question prompts the agent to answer with a numerical value.
+
+    >>> QuestionNumerical.self_check()
+
+    """
 
     question_type = "numerical"
     min_value: Optional[float] = NumericalOrNoneDescriptor()
     max_value: Optional[float] = NumericalOrNoneDescriptor()
 
+    _response_model = None
+    response_validator_class = NumericalResponseValidator
+
     def __init__(
         self,
         question_name: str,
         question_text: str,
         min_value: Optional[Union[int, float]] = None,
         max_value: Optional[Union[int, float]] = None,
+        include_comment: bool = True,
+        question_presentation: Optional[str] = None,
+        answering_instructions: Optional[str] = None,
+        permissive: bool = False,
     ):
         """Initialize the question.
 
@@ -34,30 +105,17 @@ class QuestionNumerical(QuestionBase):
         self.min_value = min_value
         self.max_value = max_value
 
+        self.include_comment = include_comment
+        self.question_presentation = question_presentation
+        self.answering_instructions = answering_instructions
+        self.permissive = permissive
+
+    def create_response_model(self):
+        return create_numeric_response(self.min_value, self.max_value, self.permissive)
+
     ################
     # Answer methods
     ################
-    def _validate_answer(
-        self, answer: dict[str, Any]
-    ) -> dict[str, Union[str, float, int]]:
-        """Validate the answer."""
-        self._validate_answer_template_basic(answer)
-        self._validate_answer_key_value_numeric(answer, "answer")
-        self._validate_answer_numerical(answer)
-        return answer
-
-    def _translate_answer_code_to_answer(self, answer, scenario: "Scenario" = None):
-        """There is no answer code."""
-        return answer
-
-    def _simulate_answer(self, human_readable: bool = True):
-        """Simulate a valid answer for debugging purposes."""
-        from edsl.utilities.utilities import random_string
-
-        return {
-            "answer": uniform(self.min_value, self.max_value),
-            "comment": random_string(),
-        }
 
     @property
     def question_html_content(self) -> str:
@@ -76,36 +134,19 @@ class QuestionNumerical(QuestionBase):
     # Helpful methods
     ################
     @classmethod
-    def example(cls) -> QuestionNumerical:
+    @inject_exception
+    def example(cls, include_comment=False) -> QuestionNumerical:
         """Return an example question."""
         return cls(
             question_name="age",
-            question_text="How old are you in years?",
+            question_text="You are a 45 year old man. How old are you in years?",
             min_value=0,
             max_value=86.7,
+            include_comment=include_comment,
         )
 
 
-def main():
-    """Show example usage."""
-    from edsl.questions.QuestionNumerical import QuestionNumerical
-
-    q = QuestionNumerical.example()
-    q.question_text
-    q.min_value
-    q.max_value
-    # validate an answer
-    q._validate_answer({"answer": 1, "comment": "I like custard"})
-    # translate answer code
-    q._translate_answer_code_to_answer(1)
-    # simulate answer
-    q._simulate_answer()
-    q._simulate_answer(human_readable=False)
-    q._validate_answer(q._simulate_answer(human_readable=False))
-    # serialization (inherits from Question)
-    q.to_dict()
-    assert q.from_dict(q.to_dict()) == q
-
+if __name__ == "__main__":
     import doctest
 
     doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/questions/QuestionRank.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 import random
+import textwrap
 from jinja2 import Template
 from typing import Any, Optional, Union
 from edsl.questions.QuestionBase import QuestionBase
@@ -10,6 +11,129 @@ from edsl.questions.descriptors import (
     NumSelectionsDescriptor,
 )
 
+from edsl.prompts import Prompt
+
+from pydantic import field_validator
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.ResponseValidatorABC import BaseResponse
+from edsl.exceptions import QuestionAnswerValidationError
+
+from pydantic import BaseModel, Field, create_model
+from typing import Optional, Any, List, Annotated, Literal
+
+
+def create_response_model(
+    choices: list,
+    num_selections: Optional[int] = None,
+    permissive: bool = False,
+):
+    """
+    :param choices: A list of allowed values for the answer field.
+    :param include_comment: Whether to include a comment field in the model.
+    :return: A new Pydantic model class.
+    """
+    # Convert the choices list to a tuple for use with Literal
+    choice_tuple = tuple(choices)
+
+    field_params = {}
+    if num_selections is not None and not permissive:
+        field_params["min_items"] = num_selections
+        field_params["max_items"] = num_selections
+
+    class RankResponse(BaseModel):
+        answer: Annotated[
+            List[Literal[choice_tuple]],
+            Field(..., **field_params),
+        ] = Field(..., description="List of selected choices")
+        comment: Optional[str] = Field(None, description="Optional comment field")
+        generated_tokens: Optional[Any] = Field(None)
+
+        class Config:
+            @staticmethod
+            def json_schema_extra(schema: dict, model: BaseModel) -> None:
+                # Add the list of choices to the schema for better documentation
+                for prop in schema.get("properties", {}).values():
+                    if prop.get("title") == "answer":
+                        prop["items"] = {"enum": choices}
+
+    return RankResponse
+
+
+class RankResponseValidator(ResponseValidatorABC):
+    required_params = ["num_selections", "permissive", "use_code", "question_options"]
+    valid_examples = []
+    invalid_examples = []
+
+    def fix(self, response, verbose=False):
+        if verbose:
+            print("Invalid response of QuestionRank was: ", False)
+        response_text = response.get("generated_tokens")
+        if response_text is None or response_text == "":  # nothing to be done
+            return response
+        # Maybe it's a comma separated list?
+        response_text = str(response.get("answer"))
+        proposed_list = (
+            response_text.replace("[", "").replace("]", "").replace("'", "").split(",")
+        )
+        proposed_list = [item.strip() for item in proposed_list]
+
+        if verbose:
+            print("Using code? ", self.use_code)
+        if self.use_code:
+            try:
+                proposed_list = [int(i) for i in proposed_list]
+            except ValueError:
+                # print("Could not convert to int")
+                pass
+
+        if verbose:
+            print("Proposed solution is: ", proposed_list)
+
+        # print(f"Ivalid generated tokens was was: {response_text}")
+        if "comment" in response:
+            proposed_data = {
+                "answer": proposed_list,
+                "comment": response["comment"],
+                "generated_tokens": response.get("generated_tokens", None),
+            }
+        else:
+            proposed_data = {
+                "answer": proposed_list,
+                "generated_tokens": response.get("generated_tokens", None),
+            }
+
+        try:
+            self.response_model(**proposed_data)
+            return proposed_data
+        except Exception as e:
+            if verbose:
+                print(f"Proposed solution {proposed_data} is invalid. Error: {e}")
+            # return response
+        if verbose:
+            print("Now seeing if responses show up in the answer")
+        matches = []
+        for index, option in enumerate(self.question_options):
+            if self.use_code:
+                if str(index) in response_text:
+                    if index not in matches:
+                        matches.append(index)
+            else:
+                if option in response_text:
+                    if option not in matches:
+                        matches.append(option)
+        proposed_data = {
+            "answer": matches,
+            "comment": response.get("comment", None),
+            "generated_tokens": response.get("generated_tokens", None),
+        }
+        try:
+            self.response_model(**proposed_data)
+            return proposed_data
+        except Exception as e:
+            if verbose:
+                print(f"Proposed solution {proposed_data} is invalid. Error: {e}")
+            return response
+
 
 class QuestionRank(QuestionBase):
     """This question prompts the agent to rank options from a list."""
@@ -18,12 +142,20 @@ class QuestionRank(QuestionBase):
     question_options: list[str] = QuestionOptionsDescriptor()
     num_selections = NumSelectionsDescriptor()
 
+    _response_model = None
+    response_validator_class = RankResponseValidator
+
     def __init__(
         self,
         question_name: str,
         question_text: str,
         question_options: list[str],
         num_selections: Optional[int] = None,
+        question_presentation: Optional[str] = None,
+        answering_instructions: Optional[str] = None,
+        permissive: bool = False,
+        use_code: bool = True,
+        include_comment: bool = True,
     ):
         """Initialize the question.
 
@@ -37,16 +169,33 @@ class QuestionRank(QuestionBase):
         self.question_text = question_text
         self.question_options = question_options
         self.num_selections = num_selections or len(question_options)
+        self.question_presentation = question_presentation
+        self.answering_instructions = answering_instructions
+        self.permissive = permissive
+        self.use_code = use_code
+        self.include_comment = include_comment
+
+    def create_response_model(self):
+        choices = (
+            self.question_options
+            if not self.use_code
+            else range(len(self.question_options))
+        )
+        return create_response_model(
+            choices=choices,
+            num_selections=self.num_selections,
+            permissive=self.permissive,
+        )
 
     ################
     # Answer methods
     ################
-    def _validate_answer(self, answer: Any) -> dict[str, list[int]]:
-        """Validate the answer."""
-        self._validate_answer_template_basic(answer)
-        self._validate_answer_key_value(answer, "answer", list)
-        self._validate_answer_rank(answer)
-        return answer
+    # def _validate_answer(self, answer: Any) -> dict[str, list[int]]:
+    #     """Validate the answer."""
+    #     self._validate_answer_template_basic(answer)
+    #     self._validate_answer_key_value(answer, "answer", list)
+    #     self._validate_answer_rank(answer)
+    #     return answer
 
     def _translate_answer_code_to_answer(
         self, answer_codes, scenario: Scenario = None
@@ -60,24 +209,27 @@ class QuestionRank(QuestionBase):
         ]
         translated_codes = []
         for answer_code in answer_codes:
-            translated_codes.append(translated_options[int(answer_code)])
+            if self._use_code:
+                translated_codes.append(translated_options[int(answer_code)])
+            else:
+                translated_codes.append(answer_code)
         return translated_codes
 
-    def _simulate_answer(self, human_readable=True) -> dict[str, Union[int, str]]:
-        """Simulate a valid answer for debugging purposes."""
-        from edsl.utilities.utilities import random_string
+    # def _simulate_answer(self, human_readable=True) -> dict[str, Union[int, str]]:
+    #     """Simulate a valid answer for debugging purposes."""
+    #     from edsl.utilities.utilities import random_string
 
-        if human_readable:
-            selected = random.sample(self.question_options, self.num_selections)
-        else:
-            selected = random.sample(
-                range(len(self.question_options)), self.num_selections
-            )
-        answer = {
-            "answer": selected,
-            "comment": random_string(),
-        }
-        return answer
+    #     if human_readable:
+    #         selected = random.sample(self.question_options, self.num_selections)
+    #     else:
+    #         selected = random.sample(
+    #             range(len(self.question_options)), self.num_selections
+    #         )
+    #     answer = {
+    #         "answer": selected,
+    #         "comment": random_string(),
+    #     }
+    #     return answer
 
     @property
     def question_html_content(self) -> str:
@@ -129,13 +281,15 @@ class QuestionRank(QuestionBase):
     # Helpful methods
     ################
     @classmethod
-    def example(cls) -> QuestionRank:
+    def example(cls, use_code=False, include_comment=True) -> QuestionRank:
         """Return an example question."""
         return cls(
             question_name="rank_foods",
             question_text="Rank your favorite foods.",
             question_options=["Pizza", "Pasta", "Salad", "Soup"],
             num_selections=2,
+            use_code=use_code,
+            include_comment=include_comment,
         )
 
 
@@ -143,7 +297,7 @@ def main():
     """Show example usage."""
     from edsl.questions.QuestionRank import QuestionRank
 
-    q = QuestionRank.example()
+    q = QuestionRank.example(use_code=True)
     q.question_text
     q.question_name
     q.question_options
@@ -152,7 +306,7 @@ def main():
     answer = {"answer": [0, 1], "comment": "I like pizza and pasta."}
     q._validate_answer(answer)
     # translate an answer code to an answer
-    q._translate_answer_code_to_answer([0, 1])
+    # q._translate_answer_code_to_answer([0, 1])
     # simulate answer
     q._simulate_answer()
     q._simulate_answer(human_readable=False)
@@ -161,6 +315,10 @@ def main():
     q.to_dict()
     assert q.from_dict(q.to_dict()) == q
 
+    q = QuestionRank.example(use_code=False)
+    answer = {"answer": ["Pizza", "Pasta"], "comment": "I like pizza and pasta."}
+    q._validate_answer(answer)
+
     import doctest
 
     doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/questions/RegisterQuestionsMeta.py

@@ -4,6 +4,8 @@ from abc import ABCMeta
 from edsl.enums import QuestionType
 from edsl.exceptions.questions import QuestionMissingTypeError, QuestionBadTypeError
 
+import inspect
+
 
 class RegisterQuestionsMeta(ABCMeta):
     """Metaclass to register output elements in a registry i.e., those that have a parent."""
@@ -13,22 +15,39 @@ class RegisterQuestionsMeta(ABCMeta):
     def __init__(cls, name, bases, dct):
         """Initialize the class and adds it to the registry if it's not the base class."""
         super(RegisterQuestionsMeta, cls).__init__(name, bases, dct)
-        if name != "QuestionBase":
+        if (
+            name != "QuestionBase"
+            and name != "QuestionFunctional"
+            and name != "QuestionAddTwoNumbers"
+        ):
             ## Enforce that all questions have a question_type class attribute
             ## and it comes from our enum of valid question types.
-            if not hasattr(cls, "question_type"):
-                raise QuestionMissingTypeError(
-                    "Question must have a question_type class attribute"
-                )
+            required_attributes = [
+                "question_type",
+                "_response_model",
+                "response_validator_class",
+            ]
+            for attr in required_attributes:
+                if not hasattr(cls, attr):
+                    raise QuestionMissingTypeError(
+                        f"Question must have a {attr} class attribute"
+                    )
 
-            if not QuestionType.is_value_valid(cls.question_type):
-                acceptable_values = [item.value for item in QuestionType]
-                raise QuestionBadTypeError(
-                    f"""question_type must be one of {QuestionType} values, which are 
-                                currently {acceptable_values}"""
-                    ""
-                )
+            init_signature = inspect.signature(cls.__init__)
+            init_params = [p for p in init_signature.parameters if p != "self"]
+            required_params = [
+                "question_presentation",
+                "answering_instructions",
+                "question_name",
+                "question_text",
+            ]
+            for param in required_params:
+                if param not in init_params:
+                    raise QuestionBadTypeError(
+                        f"Question type {name} must have a question_presentation parameter in its __init__ method"
+                    )
 
+        if name != "QuestionBase":
             RegisterQuestionsMeta._registry[name] = cls
 
     @classmethod

edsl/questions/ResponseValidatorABC.py

@@ -0,0 +1,169 @@
+from abc import ABC, abstractmethod
+from pydantic import BaseModel, Field, field_validator
+from decimal import Decimal
+from typing import Optional, Any, List, TypedDict
+
+from edsl.exceptions import QuestionAnswerValidationError
+from pydantic import ValidationError
+
+
+class BaseResponse(BaseModel):
+    answer: Any
+    comment: Optional[str] = None
+    generated_tokens: Optional[str] = None
+
+
+class ResponseValidatorABC(ABC):
+    required_params: List[str] = []
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        required_class_vars = ["required_params", "valid_examples", "invalid_examples"]
+        for var in required_class_vars:
+            if not hasattr(cls, var):
+                raise ValueError(f"Class {cls.__name__} must have a '{var}' attribute.")
+
+    def __init__(
+        self,
+        response_model: type[BaseModel],
+        exception_to_throw: Optional[Exception] = None,
+        override_answer: Optional[dict] = None,
+        **kwargs,
+    ):
+        self.response_model = response_model
+        self.exception_to_throw = exception_to_throw  # for testing
+        self.override_answer = override_answer  # for testing
+        self.original_exception = None
+
+        # Validate required parameters
+        missing_params = [
+            param for param in self.required_params if param not in kwargs
+        ]
+        if missing_params:
+            raise ValueError(
+                f"Missing required parameters: {', '.join(missing_params)}"
+            )
+
+        # Set attributes
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+        if not hasattr(self, "permissive"):
+            self.permissive = False
+
+        self.fixes_tried = 0
+
+    class RawEdslAnswerDict(TypedDict):
+        answer: Any
+        comment: Optional[str]
+        generated_tokens: Optional[str]
+
+    def _preprocess(self, data: RawEdslAnswerDict) -> RawEdslAnswerDict:
+        """This is for testing purposes. A question can be given an exception to throw or an answer to always return.
+
+        >>> rv = ResponseValidatorABC.example()
+        >>> rv.override_answer = {"answer": 42}
+        >>> rv.validate({"answer": 23})
+        {'answer': Decimal('42'), 'comment': None, 'generated_tokens': None}
+        """
+        if self.exception_to_throw:
+            raise self.exception_to_throw
+        return self.override_answer if self.override_answer else data
+
+    def _base_validate(self, data: RawEdslAnswerDict) -> BaseModel:
+        """This is the main validation function. It takes the response_model and checks the data against it, returning the instantiated model.
+
+        >>> rv = ResponseValidatorABC.example("numerical")
+        >>> rv._base_validate({"answer": 42})
+        ConstrainedNumericResponse(answer=Decimal('42'), comment=None, generated_tokens=None)
+        """
+        try:
+            return self.response_model(**data)
+        except ValidationError as e:
+            raise QuestionAnswerValidationError(e, data=data, model=self.response_model)
+
+    def post_validation_answer_convert(self, data):
+        return data
+
+    class EdslAnswerDict(TypedDict):
+        answer: Any
+        comment: Optional[str]
+        generated_tokens: Optional[str]
+
+    def validate(
+        self, raw_edsl_answer_dict: RawEdslAnswerDict, fix=False, verbose=False
+    ) -> EdslAnswerDict:
+        """This is the main validation function.
+
+        >>> rv = ResponseValidatorABC.example("numerical")
+        >>> rv.validate({"answer": 42})
+        {'answer': Decimal('42'), 'comment': None, 'generated_tokens': None}
+        >>> rv.max_value
+        86.7
+        >>> rv.validate({"answer": "120"})
+        Traceback (most recent call last):
+        ...
+        edsl.exceptions.questions.QuestionAnswerValidationError:...
+        >>> from edsl import QuestionNumerical
+        >>> q = QuestionNumerical.example()
+        >>> q.permissive = True
+        >>> rv = q.response_validator
+        >>> rv.validate({"answer": "120"})
+        {'answer': Decimal('120'), 'comment': None, 'generated_tokens': None}
+        >>> rv.validate({"answer": "poo"})
+        Traceback (most recent call last):
+        ...
+        edsl.exceptions.questions.QuestionAnswerValidationError:...
+        """
+        proposed_edsl_answer_dict = self._preprocess(raw_edsl_answer_dict)
+        try:
+            pydantic_edsl_answer: BaseModel = self._base_validate(
+                proposed_edsl_answer_dict
+            )
+            edsl_answer_dict = self._extract_answer(pydantic_edsl_answer)
+            return self._post_process(edsl_answer_dict)
+        except QuestionAnswerValidationError as e:
+            if verbose:
+                print(f"Failed to validate {raw_edsl_answer_dict}; {str(e)}")
+            return self._handle_exception(e, raw_edsl_answer_dict)
+
+    def _handle_exception(self, e: Exception, raw_edsl_answer_dict) -> EdslAnswerDict:
+        if self.fixes_tried == 0:
+            self.original_exception = e
+
+        if self.fixes_tried == 0 and hasattr(self, "fix"):
+            self.fixes_tried += 1
+            fixed_data = self.fix(raw_edsl_answer_dict)
+            try:
+                return self.validate(fixed_data, fix=True)
+            except Exception as e:
+                pass  # we don't log failed fixes
+
+        raise QuestionAnswerValidationError(
+            self.original_exception,
+            data=raw_edsl_answer_dict,
+            model=self.response_model,
+        )
+
+    def _check_constraints(self, pydantic_edsl_answer: BaseModel) -> dict:
+        pass
+
+    def _extract_answer(self, response: BaseModel) -> EdslAnswerDict:
+        return response.model_dump()
+
+    def _post_process(self, edsl_answer_dict: EdslAnswerDict) -> EdslAnswerDict:
+        return edsl_answer_dict
+
+    @classmethod
+    def example(cls, question_type="numerical"):
+        from edsl import Question
+
+        q = Question.example(question_type)
+        return q.response_validator
+
+
+# Example usage
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)