edsl 0.1.39.dev2__py3-none-any.whl → 0.1.39.dev3__py3-none-any.whl

edsl/questions/Quick.py

@@ -1,41 +1,41 @@
-from edsl import (
-    QuestionFreeText,
-    QuestionMultipleChoice,
-    Survey,
-    QuestionList,
-    Question,
-)
-
-
-def Quick(question_text):
-    q_type = QuestionMultipleChoice(
-        question_text=f"A researcher is asking a language model this: {question_text}. What is the most appropriate type of question to ask?",
-        question_name="potential_question_type",
-        question_options=["multiple_choice", "list", "free_text"],
-    )
-
-    q_name = QuestionFreeText(
-        question_text=f"A researcher is asking a language model this: {question_text}. What is a good name for this question that's a valid python identifier? Just return the proposed identifer",
-        question_name="potential_question_name",
-    )
-
-    q_options = QuestionList(
-        question_text=f"A research is asking this question: { question_text }. What are the possible options for this question?",
-        question_name="potential_question_options",
-    )
-
-    survey = Survey([q_type, q_name, q_options]).add_skip_rule(
-        q_options, "{{ potential_question_type }} != 'multiple_choice'"
-    )
-    return survey
-    # results = survey.run()
-    # question_type = results.select("potential_question_type").first()
-    # question_options = results.select("potential_question_options").first()
-    # question_name = results.select("potential_question_name").first()
-    # print("Question Type: ", question_type)
-    # print("Question Name: ", question_name)
-    # print("Question Options: ", question_options)
-    # if question_options == None:
-    #     return Question(question_type, question_name = question_name)
-    # else:
-    #     return Question(question_type, question_name = question_name, question_options = question_options)
+from edsl import (
+    QuestionFreeText,
+    QuestionMultipleChoice,
+    Survey,
+    QuestionList,
+    Question,
+)
+
+
+def Quick(question_text):
+    q_type = QuestionMultipleChoice(
+        question_text=f"A researcher is asking a language model this: {question_text}. What is the most appropriate type of question to ask?",
+        question_name="potential_question_type",
+        question_options=["multiple_choice", "list", "free_text"],
+    )
+
+    q_name = QuestionFreeText(
+        question_text=f"A researcher is asking a language model this: {question_text}. What is a good name for this question that's a valid python identifier? Just return the proposed identifer",
+        question_name="potential_question_name",
+    )
+
+    q_options = QuestionList(
+        question_text=f"A research is asking this question: { question_text }. What are the possible options for this question?",
+        question_name="potential_question_options",
+    )
+
+    survey = Survey([q_type, q_name, q_options]).add_skip_rule(
+        q_options, "{{ potential_question_type }} != 'multiple_choice'"
+    )
+    return survey
+    # results = survey.run()
+    # question_type = results.select("potential_question_type").first()
+    # question_options = results.select("potential_question_options").first()
+    # question_name = results.select("potential_question_name").first()
+    # print("Question Type: ", question_type)
+    # print("Question Name: ", question_name)
+    # print("Question Options: ", question_options)
+    # if question_options == None:
+    #     return Question(question_type, question_name = question_name)
+    # else:
+    #     return Question(question_type, question_name = question_name, question_options = question_options)

edsl/questions/RegisterQuestionsMeta.py

@@ -1,71 +1,71 @@
-from __future__ import annotations
-from abc import ABCMeta
-
-from edsl.enums import QuestionType
-from edsl.exceptions.questions import QuestionMissingTypeError, QuestionBadTypeError
-
-import inspect
-
-
-class RegisterQuestionsMeta(ABCMeta):
-    """Metaclass to register output elements in a registry i.e., those that have a parent."""
-
-    _registry = {}  # Initialize the registry as a dictionary
-
-    def __init__(cls, name, bases, dct):
-        """Initialize the class and adds it to the registry if it's not the base class."""
-        super(RegisterQuestionsMeta, cls).__init__(name, bases, dct)
-        if (
-            name != "QuestionBase"
-            and name != "QuestionFunctional"
-            and name != "QuestionAddTwoNumbers"
-        ):
-            ## Enforce that all questions have a question_type class attribute
-            ## and it comes from our enum of valid question types.
-            required_attributes = [
-                "question_type",
-                "_response_model",
-                "response_validator_class",
-            ]
-            for attr in required_attributes:
-                if not hasattr(cls, attr):
-                    raise QuestionMissingTypeError(
-                        f"Question must have a {attr} class attribute"
-                    )
-
-            init_signature = inspect.signature(cls.__init__)
-            init_params = [p for p in init_signature.parameters if p != "self"]
-            required_params = [
-                "question_presentation",
-                "answering_instructions",
-                "question_name",
-                "question_text",
-            ]
-            for param in required_params:
-                if param not in init_params:
-                    raise QuestionBadTypeError(
-                        f"Question type {name} must have a question_presentation parameter in its __init__ method"
-                    )
-
-        if name != "QuestionBase":
-            RegisterQuestionsMeta._registry[name] = cls
-
-    @classmethod
-    def get_registered_classes(cls):
-        """Return the registry of registered classes."""
-        return cls._registry
-
-    @classmethod
-    def question_types_to_classes(
-        cls,
-    ):
-        """Return a dictionary of question types to classes."""
-        d = {}
-        for classname, cls in cls._registry.items():
-            if hasattr(cls, "question_type"):
-                d[cls.question_type] = cls
-            else:
-                raise Exception(
-                    f"Class {classname} does not have a question_type class attribute"
-                )
-        return d
+from __future__ import annotations
+from abc import ABCMeta
+
+from edsl.enums import QuestionType
+from edsl.exceptions.questions import QuestionMissingTypeError, QuestionBadTypeError
+
+import inspect
+
+
+class RegisterQuestionsMeta(ABCMeta):
+    """Metaclass to register output elements in a registry i.e., those that have a parent."""
+
+    _registry = {}  # Initialize the registry as a dictionary
+
+    def __init__(cls, name, bases, dct):
+        """Initialize the class and adds it to the registry if it's not the base class."""
+        super(RegisterQuestionsMeta, cls).__init__(name, bases, dct)
+        if (
+            name != "QuestionBase"
+            and name != "QuestionFunctional"
+            and name != "QuestionAddTwoNumbers"
+        ):
+            ## Enforce that all questions have a question_type class attribute
+            ## and it comes from our enum of valid question types.
+            required_attributes = [
+                "question_type",
+                "_response_model",
+                "response_validator_class",
+            ]
+            for attr in required_attributes:
+                if not hasattr(cls, attr):
+                    raise QuestionMissingTypeError(
+                        f"Question must have a {attr} class attribute"
+                    )
+
+            init_signature = inspect.signature(cls.__init__)
+            init_params = [p for p in init_signature.parameters if p != "self"]
+            required_params = [
+                "question_presentation",
+                "answering_instructions",
+                "question_name",
+                "question_text",
+            ]
+            for param in required_params:
+                if param not in init_params:
+                    raise QuestionBadTypeError(
+                        f"Question type {name} must have a question_presentation parameter in its __init__ method"
+                    )
+
+        if name != "QuestionBase":
+            RegisterQuestionsMeta._registry[name] = cls
+
+    @classmethod
+    def get_registered_classes(cls):
+        """Return the registry of registered classes."""
+        return cls._registry
+
+    @classmethod
+    def question_types_to_classes(
+        cls,
+    ):
+        """Return a dictionary of question types to classes."""
+        d = {}
+        for classname, cls in cls._registry.items():
+            if hasattr(cls, "question_type"):
+                d[cls.question_type] = cls
+            else:
+                raise Exception(
+                    f"Class {classname} does not have a question_type class attribute"
+                )
+        return d

edsl/questions/ResponseValidatorABC.py

@@ -1,200 +1,174 @@
-import logging
-from abc import ABC, abstractmethod
-from typing import Optional, Any, List, TypedDict
-
-from pydantic import BaseModel, Field, field_validator, ValidationError
-
-from edsl.exceptions.questions import QuestionAnswerValidationError
-from edsl.questions.ExceptionExplainer import ExceptionExplainer
-
-
-class BaseResponse(BaseModel):
-    answer: Any
-    comment: Optional[str] = None
-    generated_tokens: Optional[str] = None
-
-
-class ResponseValidatorABC(ABC):
-    required_params: List[str] = []
-
-    def __init_subclass__(cls, **kwargs):
-        super().__init_subclass__(**kwargs)
-        required_class_vars = ["required_params", "valid_examples", "invalid_examples"]
-        for var in required_class_vars:
-            if not hasattr(cls, var):
-                raise ValueError(f"Class {cls.__name__} must have a '{var}' attribute.")
-
-    def __init__(
-        self,
-        response_model: type[BaseModel],
-        exception_to_throw: Optional[Exception] = None,
-        override_answer: Optional[dict] = None,
-        **kwargs,
-    ):
-        self.response_model = response_model
-        self.exception_to_throw = exception_to_throw  # for testing
-        self.override_answer = override_answer  # for testing
-        self.original_exception = None
-
-        # Validate required parameters
-        missing_params = [
-            param for param in self.required_params if param not in kwargs
-        ]
-        if missing_params:
-            raise ValueError(
-                f"Missing required parameters: {', '.join(missing_params)}"
-            )
-
-        # Set attributes
-        for key, value in kwargs.items():
-            setattr(self, key, value)
-
-        if not hasattr(self, "permissive"):
-            self.permissive = False
-
-        self.fixes_tried = 0
-
-    class RawEdslAnswerDict(TypedDict):
-        answer: Any
-        comment: Optional[str]
-        generated_tokens: Optional[str]
-
-    def _preprocess(self, data: RawEdslAnswerDict) -> RawEdslAnswerDict:
-        """This is for testing purposes. A question can be given an exception to throw or an answer to always return.
-
-        >>> rv = ResponseValidatorABC.example()
-        >>> rv.override_answer = {"answer": 42}
-        >>> rv.validate({"answer": 23})
-        {'answer': 42, 'comment': None, 'generated_tokens': None}
-        """
-        if self.exception_to_throw:
-            raise self.exception_to_throw
-        return self.override_answer if self.override_answer else data
-
-    def _base_validate(self, data: RawEdslAnswerDict) -> BaseModel:
-        """This is the main validation function. It takes the response_model and checks the data against it,
-        returning the instantiated model.
-
-        >>> rv = ResponseValidatorABC.example("numerical")
-        >>> rv._base_validate({"answer": 42})
-        ConstrainedNumericResponse(answer=42, comment=None, generated_tokens=None)
-        """
-        try:
-            return self.response_model(**data)
-        except ValidationError as e:
-            raise QuestionAnswerValidationError(
-                message=str(e), pydantic_error=e, data=data, model=self.response_model
-            )
-
-    def post_validation_answer_convert(self, data):
-        return data
-
-    class EdslAnswerDict(TypedDict):
-        answer: Any
-        comment: Optional[str]
-        generated_tokens: Optional[str]
-
-    def validate(
-        self,
-        raw_edsl_answer_dict: RawEdslAnswerDict,
-        fix=False,
-        verbose=False,
-        replacement_dict: dict = None,
-    ) -> EdslAnswerDict:
-        """This is the main validation function.
-
-        >>> rv = ResponseValidatorABC.example("numerical")
-        >>> rv.validate({"answer": 42})
-        {'answer': 42, 'comment': None, 'generated_tokens': None}
-        >>> rv.max_value
-        86.7
-        >>> rv.validate({"answer": "120"})
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionAnswerValidationError:...
-        >>> from edsl import QuestionNumerical
-        >>> q = QuestionNumerical.example()
-        >>> q.permissive = True
-        >>> rv = q.response_validator
-        >>> rv.validate({"answer": "120"})
-        {'answer': 120, 'comment': None, 'generated_tokens': None}
-        >>> rv.validate({"answer": "poo"})
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionAnswerValidationError:...
-        """
-        proposed_edsl_answer_dict = self._preprocess(raw_edsl_answer_dict)
-        try:
-            pydantic_edsl_answer: BaseModel = self._base_validate(
-                proposed_edsl_answer_dict
-            )
-            edsl_answer_dict = self._extract_answer(pydantic_edsl_answer)
-            return self._post_process(edsl_answer_dict)
-        except QuestionAnswerValidationError as e:
-            return self._handle_exception(e, raw_edsl_answer_dict)
-
-    def human_explanation(self, e: QuestionAnswerValidationError):
-        explanation = ExceptionExplainer(e, model_response=e.data).explain()
-        return explanation
-        # return e
-
-    def _handle_exception(self, e: Exception, raw_edsl_answer_dict) -> EdslAnswerDict:
-        if self.fixes_tried == 0:
-            self.original_exception = e
-
-        if self.fixes_tried == 0 and hasattr(self, "fix"):
-            self.fixes_tried += 1
-            fixed_data = self.fix(raw_edsl_answer_dict)
-            try:
-                return self.validate(fixed_data, fix=True)  # early return if validates
-            except Exception as e:
-                pass  # we don't log failed fixes
-
-        # If the exception is already a QuestionAnswerValidationError, raise it
-        if isinstance(self.original_exception, QuestionAnswerValidationError):
-            raise self.original_exception
-
-        # If nothing worked, raise the original exception
-        raise QuestionAnswerValidationError(
-            message=self.original_exception,
-            pydantic_error=self.original_exception,
-            data=raw_edsl_answer_dict,
-            model=self.response_model,
-        )
-
-    def _check_constraints(self, pydantic_edsl_answer: BaseModel) -> dict:
-        pass
-
-    def _extract_answer(self, response: BaseModel) -> EdslAnswerDict:
-        return response.model_dump()
-
-    def _post_process(self, edsl_answer_dict: EdslAnswerDict) -> EdslAnswerDict:
-        return edsl_answer_dict
-
-    @classmethod
-    def example(cls, question_type="numerical"):
-        from edsl import Question
-
-        q = Question.example(question_type)
-        return q.response_validator
-
-
-def main():
-    rv = ResponseValidatorABC.example()
-    print(rv.validate({"answer": 42}))
-
-
-# Example usage
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)
-
-    rv = ResponseValidatorABC.example()
-    # print(rv.validate({"answer": 42}))
-
-    rv = ResponseValidatorABC.example()
-    try:
-        rv.validate({"answer": "120"})
-    except QuestionAnswerValidationError as e:
-        print(rv.human_explanation(e))
+from abc import ABC, abstractmethod
+from pydantic import BaseModel, Field, field_validator
+
+# from decimal import Decimal
+from typing import Optional, Any, List, TypedDict
+
+from edsl.exceptions import QuestionAnswerValidationError
+from pydantic import ValidationError
+
+
+class BaseResponse(BaseModel):
+    answer: Any
+    comment: Optional[str] = None
+    generated_tokens: Optional[str] = None
+
+
+class ResponseValidatorABC(ABC):
+    required_params: List[str] = []
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        required_class_vars = ["required_params", "valid_examples", "invalid_examples"]
+        for var in required_class_vars:
+            if not hasattr(cls, var):
+                raise ValueError(f"Class {cls.__name__} must have a '{var}' attribute.")
+
+    def __init__(
+        self,
+        response_model: type[BaseModel],
+        exception_to_throw: Optional[Exception] = None,
+        override_answer: Optional[dict] = None,
+        **kwargs,
+    ):
+        self.response_model = response_model
+        self.exception_to_throw = exception_to_throw  # for testing
+        self.override_answer = override_answer  # for testing
+        self.original_exception = None
+
+        # Validate required parameters
+        missing_params = [
+            param for param in self.required_params if param not in kwargs
+        ]
+        if missing_params:
+            raise ValueError(
+                f"Missing required parameters: {', '.join(missing_params)}"
+            )
+
+        # Set attributes
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+        if not hasattr(self, "permissive"):
+            self.permissive = False
+
+        self.fixes_tried = 0
+
+    class RawEdslAnswerDict(TypedDict):
+        answer: Any
+        comment: Optional[str]
+        generated_tokens: Optional[str]
+
+    def _preprocess(self, data: RawEdslAnswerDict) -> RawEdslAnswerDict:
+        """This is for testing purposes. A question can be given an exception to throw or an answer to always return.
+
+        >>> rv = ResponseValidatorABC.example()
+        >>> rv.override_answer = {"answer": 42}
+        >>> rv.validate({"answer": 23})
+        {'answer': 42, 'comment': None, 'generated_tokens': None}
+        """
+        if self.exception_to_throw:
+            raise self.exception_to_throw
+        return self.override_answer if self.override_answer else data
+
+    def _base_validate(self, data: RawEdslAnswerDict) -> BaseModel:
+        """This is the main validation function. It takes the response_model and checks the data against it, returning the instantiated model.
+
+        >>> rv = ResponseValidatorABC.example("numerical")
+        >>> rv._base_validate({"answer": 42})
+        ConstrainedNumericResponse(answer=42, comment=None, generated_tokens=None)
+        """
+        try:
+            return self.response_model(**data)
+        except ValidationError as e:
+            raise QuestionAnswerValidationError(e, data=data, model=self.response_model)
+
+    def post_validation_answer_convert(self, data):
+        return data
+
+    class EdslAnswerDict(TypedDict):
+        answer: Any
+        comment: Optional[str]
+        generated_tokens: Optional[str]
+
+    def validate(
+        self,
+        raw_edsl_answer_dict: RawEdslAnswerDict,
+        fix=False,
+        verbose=False,
+        replacement_dict: dict = None,
+    ) -> EdslAnswerDict:
+        """This is the main validation function.
+
+        >>> rv = ResponseValidatorABC.example("numerical")
+        >>> rv.validate({"answer": 42})
+        {'answer': 42, 'comment': None, 'generated_tokens': None}
+        >>> rv.max_value
+        86.7
+        >>> rv.validate({"answer": "120"})
+        Traceback (most recent call last):
+        ...
+        edsl.exceptions.questions.QuestionAnswerValidationError:...
+        >>> from edsl import QuestionNumerical
+        >>> q = QuestionNumerical.example()
+        >>> q.permissive = True
+        >>> rv = q.response_validator
+        >>> rv.validate({"answer": "120"})
+        {'answer': 120, 'comment': None, 'generated_tokens': None}
+        >>> rv.validate({"answer": "poo"})
+        Traceback (most recent call last):
+        ...
+        edsl.exceptions.questions.QuestionAnswerValidationError:...
+        """
+        proposed_edsl_answer_dict = self._preprocess(raw_edsl_answer_dict)
+        try:
+            pydantic_edsl_answer: BaseModel = self._base_validate(
+                proposed_edsl_answer_dict
+            )
+            edsl_answer_dict = self._extract_answer(pydantic_edsl_answer)
+            return self._post_process(edsl_answer_dict)
+        except QuestionAnswerValidationError as e:
+            if verbose:
+                print(f"Failed to validate {raw_edsl_answer_dict}; {str(e)}")
+            return self._handle_exception(e, raw_edsl_answer_dict)
+
+    def _handle_exception(self, e: Exception, raw_edsl_answer_dict) -> EdslAnswerDict:
+        if self.fixes_tried == 0:
+            self.original_exception = e
+
+        if self.fixes_tried == 0 and hasattr(self, "fix"):
+            self.fixes_tried += 1
+            fixed_data = self.fix(raw_edsl_answer_dict)
+            try:
+                return self.validate(fixed_data, fix=True)
+            except Exception as e:
+                pass  # we don't log failed fixes
+
+        raise QuestionAnswerValidationError(
+            self.original_exception,
+            data=raw_edsl_answer_dict,
+            model=self.response_model,
+        )
+
+    def _check_constraints(self, pydantic_edsl_answer: BaseModel) -> dict:
+        pass
+
+    def _extract_answer(self, response: BaseModel) -> EdslAnswerDict:
+        return response.model_dump()
+
+    def _post_process(self, edsl_answer_dict: EdslAnswerDict) -> EdslAnswerDict:
+        return edsl_answer_dict
+
+    @classmethod
+    def example(cls, question_type="numerical"):
+        from edsl import Question
+
+        q = Question.example(question_type)
+        return q.response_validator
+
+
+# Example usage
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)