edsl 0.1.14__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/questions/response_validator_abc.py

@@ -0,0 +1,188 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Any, List, TypedDict
+
+from pydantic import BaseModel, Field, field_validator, ValidationError
+
+from edsl.exceptions.questions import QuestionAnswerValidationError
+from edsl.questions.ExceptionExplainer import ExceptionExplainer
+
+from edsl.questions.data_structures import (
+    RawEdslAnswerDict,
+    EdslAnswerDict,
+)
+
+
+class ResponseValidatorABC(ABC):
+    required_params: List[str] = []
+
+    def __init_subclass__(cls, **kwargs):
+        """This is a metaclass that ensures that all subclasses of ResponseValidatorABC have the required class variables."""
+        super().__init_subclass__(**kwargs)
+        required_class_vars = ["required_params", "valid_examples", "invalid_examples"]
+        for var in required_class_vars:
+            if not hasattr(cls, var):
+                raise ValueError(f"Class {cls.__name__} must have a '{var}' attribute.")
+
+    def __init__(
+        self,
+        response_model: type[BaseModel],
+        exception_to_throw: Optional[Exception] = None,
+        override_answer: Optional[dict] = None,
+        **kwargs,
+    ):
+        self.response_model = response_model
+        self.exception_to_throw = exception_to_throw  # for testing
+        self.override_answer = override_answer  # for testing
+        self.original_exception = None
+
+        # Validate required parameters
+        missing_params = [
+            param for param in self.required_params if param not in kwargs
+        ]
+        if missing_params:
+            raise ValueError(
+                f"Missing required parameters: {', '.join(missing_params)}"
+            )
+
+        # Set attributes
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+        if not hasattr(self, "permissive"):
+            self.permissive = False
+
+        self.fixes_tried = 0  # how many times we've tried to fix the answer
+
+    def _preprocess(self, data: RawEdslAnswerDict) -> RawEdslAnswerDict:
+        """This is for testing purposes. A question can be given an exception to throw or an answer to always return.
+
+        >>> rv = ResponseValidatorABC.example()
+        >>> rv.override_answer = {"answer": 42}
+        >>> rv.validate({"answer": 23})
+        {'answer': 42, 'comment': None, 'generated_tokens': None}
+        """
+        if self.exception_to_throw:
+            raise self.exception_to_throw
+        return self.override_answer if self.override_answer else data
+
+    def _base_validate(self, data: RawEdslAnswerDict) -> BaseModel:
+        """This is the main validation function. It takes the response_model and checks the data against it,
+        returning the instantiated model.
+
+        >>> rv = ResponseValidatorABC.example("numerical")
+        >>> rv._base_validate({"answer": 42})
+        ConstrainedNumericResponse(answer=42, comment=None, generated_tokens=None)
+        """
+        try:
+            return self.response_model(**data)
+        except ValidationError as e:
+            raise QuestionAnswerValidationError(
+                message=str(e), pydantic_error=e, data=data, model=self.response_model
+            )
+
+    def post_validation_answer_convert(self, data):
+        return data
+
+    def validate(
+        self,
+        raw_edsl_answer_dict: RawEdslAnswerDict,
+        fix=False,
+        verbose=False,
+        replacement_dict: dict = None,
+    ) -> EdslAnswerDict:
+        """This is the main validation function.
+
+        >>> rv = ResponseValidatorABC.example("numerical")
+        >>> rv.validate({"answer": 42})
+        {'answer': 42, 'comment': None, 'generated_tokens': None}
+        >>> rv.max_value
+        86.7
+        >>> rv.validate({"answer": "120"})
+        Traceback (most recent call last):
+        ...
+        edsl.exceptions.questions.QuestionAnswerValidationError:...
+        >>> from edsl import QuestionNumerical
+        >>> q = QuestionNumerical.example()
+        >>> q.permissive = True
+        >>> rv = q.response_validator
+        >>> rv.validate({"answer": "120"})
+        {'answer': 120, 'comment': None, 'generated_tokens': None}
+        >>> rv.validate({"answer": "poo"})
+        Traceback (most recent call last):
+        ...
+        edsl.exceptions.questions.QuestionAnswerValidationError:...
+        """
+        proposed_edsl_answer_dict = self._preprocess(raw_edsl_answer_dict)
+        try:
+            pydantic_edsl_answer: BaseModel = self._base_validate(
+                proposed_edsl_answer_dict
+            )
+            edsl_answer_dict = self._extract_answer(pydantic_edsl_answer)
+            return self._post_process(edsl_answer_dict)
+        except QuestionAnswerValidationError as e:
+            return self._handle_exception(e, raw_edsl_answer_dict)
+
+    def human_explanation(self, e: QuestionAnswerValidationError):
+        explanation = ExceptionExplainer(e, model_response=e.data).explain()
+        return explanation
+
+    def _handle_exception(self, e: Exception, raw_edsl_answer_dict) -> EdslAnswerDict:
+        if self.fixes_tried == 0:
+            self.original_exception = e
+
+        if self.fixes_tried == 0 and hasattr(self, "fix"):
+            self.fixes_tried += 1
+            fixed_data = self.fix(raw_edsl_answer_dict)
+            try:
+                return self.validate(fixed_data, fix=True)  # early return if validates
+            except Exception as e:
+                pass  # we don't log failed fixes
+
+        # If the exception is already a QuestionAnswerValidationError, raise it
+        if isinstance(self.original_exception, QuestionAnswerValidationError):
+            raise self.original_exception
+
+        # If nothing worked, raise the original exception
+        raise QuestionAnswerValidationError(
+            message=self.original_exception,
+            pydantic_error=self.original_exception,
+            data=raw_edsl_answer_dict,
+            model=self.response_model,
+        )
+
+    def _check_constraints(self, pydantic_edsl_answer: BaseModel) -> dict:
+        pass
+
+    def _extract_answer(self, response: BaseModel) -> EdslAnswerDict:
+        return response.model_dump()
+
+    def _post_process(self, edsl_answer_dict: EdslAnswerDict) -> EdslAnswerDict:
+        return edsl_answer_dict
+
+    @classmethod
+    def example(cls, question_type="numerical"):
+        from edsl import Question
+
+        q = Question.example(question_type)
+        return q.response_validator
+
+
+def main():
+    rv = ResponseValidatorABC.example()
+    print(rv.validate({"answer": 42}))
+
+
+# Example usage
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)
+
+    rv = ResponseValidatorABC.example()
+    # print(rv.validate({"answer": 42}))
+
+    rv = ResponseValidatorABC.example()
+    try:
+        rv.validate({"answer": "120"})
+    except QuestionAnswerValidationError as e:
+        print(rv.human_explanation(e))

edsl/questions/response_validator_factory.py

@@ -0,0 +1,34 @@
+from edsl.questions.data_structures import BaseModel
+from edsl.questions.response_validator_abc import ResponseValidatorABC
+
+
+class ResponseValidatorFactory:
+    """Factory class to create a response validator for a question."""
+
+    def __init__(self, question):
+        self.question = question
+
+    @property
+    def response_model(self) -> type["BaseModel"]:
+        if self.question._response_model is not None:
+            return self.question._response_model
+        else:
+            return self.question.create_response_model()
+
+    @property
+    def response_validator(self) -> "ResponseValidatorABC":
+        """Return the response validator."""
+        params = (
+            {
+                "response_model": self.question.response_model,
+            }
+            | {k: getattr(self.question, k) for k in self.validator_parameters}
+            | {"exception_to_throw": getattr(self.question, "exception_to_throw", None)}
+            | {"override_answer": getattr(self.question, "override_answer", None)}
+        )
+        return self.question.response_validator_class(**params)
+
+    @property
+    def validator_parameters(self) -> list[str]:
+        """Return the parameters required for the response validator."""
+        return self.question.response_validator_class.required_params

edsl/questions/settings.py

@@ -1,9 +1,12 @@
+"""Settings for the questions module."""
+
+
 class Settings:
     """Settings for the questions module."""
 
     MAX_ANSWER_LENGTH = 2000
     MAX_EXPRESSION_CONSTRAINT_LENGTH = 1000
-    MAX_NUM_OPTIONS = 20
+    MAX_NUM_OPTIONS = 200
     MIN_NUM_OPTIONS = 2
-    MAX_OPTION_LENGTH = 1000
+    MAX_OPTION_LENGTH = 10000
     MAX_QUESTION_LENGTH = 100000

edsl/questions/templates/budget/answering_instructions.jinja

@@ -0,0 +1,7 @@
+Return only a comma-separated list the values in the same order as the options, with 0s included, on one line, in square braces.
+
+Example: if there are 4 options, the response should be "[25,25,25,25]" to allocate 25 to each option.
+
+{% if include_comment %}
+After the answer, you can put a comment explaining your choice on the next line.
+{% endif %}

edsl/questions/templates/budget/question_presentation.jinja

@@ -0,0 +1,7 @@
+{{question_text}}
+The options are 
+{% for option in question_options %}
+{{ loop.index0 }}: {{option}}
+{% endfor %}                       
+Allocate your budget of {{budget_sum}} among the options. 
+

edsl/questions/templates/checkbox/answering_instructions.jinja

@@ -0,0 +1,10 @@
+{# Answering Instructions #}
+{% if use_code %}
+Please respond only with a comma-separated list of the code of the options that apply, with square brackets. E.g., [0, 1, 3]
+{% else %}
+Please respond only with a comma-separated list of the options that apply, with square brackets. E.g., ['Good', 'Bad', 'Ugly']
+{% endif %}
+{% if include_comment %}
+After the answer, you can put a comment explaining your choice on the next line.
+{% endif %}
+

edsl/questions/templates/checkbox/question_presentation.jinja

@@ -0,0 +1,22 @@
+{{question_text}}
+{% if use_code %}
+    {% for option in question_options %}
+{{ loop.index0 }}: {{option}}
+    {% endfor %}
+{% else %}
+    {% for option in question_options %}
+{{ option }}
+    {% endfor %}    
+{% endif %}
+
+{# Restrictions #}
+{% if min_selections != None and max_selections != None and min_selections == max_selections %}
+You must select exactly {{min_selections}} options.
+{% elif min_selections != None and max_selections != None %}
+Minimum number of options that must be selected: {{min_selections}}.
+Maximum number of options that must be selected: {{max_selections}}.
+{% elif min_selections != None %}
+Minimum number of options that must be selected: {{min_selections}}.
+{% elif max_selections != None %}
+Maximum number of options that must be selected: {{max_selections}}.
+{% endif %}

edsl/questions/templates/dict/answering_instructions.jinja

@@ -0,0 +1,21 @@
+Please respond with a dictionary using the following keys: {{ answer_keys | join(', ') }}.
+
+{% if value_descriptions %}
+Here are descriptions of the values to provide:
+{% for idx in range(answer_keys | length) %}
+- "{{ answer_keys[idx] }}": "{{ value_descriptions[idx] }}"
+{% endfor %}
+{% endif %}
+
+{% if value_types %}
+The values should be formatted in the following types:
+{% for idx in range(answer_keys | length) %}
+- "{{ answer_keys[idx] }}": "{{ value_types[idx] }}"
+{% endfor %}
+{% endif %}
+
+If you do not have a value for a given key, use "null".
+
+{% if include_comment %}
+After the answer, you can put a comment explaining your response on the next line.
+{% endif %}

edsl/questions/templates/dict/question_presentation.jinja

@@ -0,0 +1 @@
+{{question_text}}

edsl/questions/templates/extract/answering_instructions.jinja

@@ -0,0 +1,7 @@
+An ANSWER should be formatted like this: 
+
+{{ answer_template }}
+
+It should have the same keys but values extracted from the input.
+If the value of a key is not present in the input, fill with "null".
+Put any comments in the next line after the answer.

edsl/questions/templates/extract/question_presentation.jinja

@@ -0,0 +1 @@
+{{question_text}}

edsl/questions/templates/free_text/question_presentation.jinja

@@ -0,0 +1 @@
+{{question_text}}

edsl/questions/templates/likert_five/answering_instructions.jinja

@@ -0,0 +1,10 @@
+{# Answering Instructions #}
+{% if use_code %}
+Respond only with the code corresponding to one of the options.
+{% else %}
+Respond only with a string corresponding to one of the options.
+{% endif %}
+{% if include_comment %}
+After the answer, you can put a comment explaining why you chose that option on the next line.
+{% endif %}
+

edsl/questions/templates/likert_five/question_presentation.jinja

@@ -0,0 +1,12 @@
+{# Question Presention #}
+{{question_text}}
+{% if use_code %}
+    {%- for option in question_options %}
+{{ loop.index0 }}: {{option}}
+    {% endfor %}
+{% else %}
+    {% for option in question_options %}
+{{option}}
+    {% endfor %}
+{% endif %}
+Only 1 option may be selected.

edsl/questions/templates/linear_scale/answering_instructions.jinja

@@ -0,0 +1,5 @@
+{# Answering Instructions #}
+Respond only with the code corresponding to one of the options. E.g., "1" or "5" by itself.
+{% if include_comment %}
+After the answer, you can put a comment explaining why you chose that option on the next line.
+{% endif %}

edsl/questions/templates/linear_scale/question_presentation.jinja

@@ -0,0 +1,5 @@
+{{question_text}}
+{% for option in question_options %}
+{{option}} : {{ option_labels.get(option, "") }}
+{% endfor %}
+Only 1 option may be selected.

edsl/questions/templates/list/answering_instructions.jinja

@@ -0,0 +1,4 @@
+Return your answers on one line, in a comma-separated list of your responses, with square brackets and each answer in quotes E.g., ["A", "B", "C"]
+{% if include_comment %}
+After the answers, you can put a comment explaining your choice on the next line.
+{% endif %}

edsl/questions/templates/list/question_presentation.jinja

@@ -0,0 +1,5 @@
+{{question_text}}
+
+{% if max_list_items is not none %}
+The list must not contain more than {{ max_list_items }} items.
+{% endif %}

edsl/questions/templates/matrix/__init__.py

@@ -0,0 +1 @@
+

edsl/questions/templates/matrix/answering_instructions.jinja

@@ -0,0 +1,5 @@
+Please respond with a dictionary mapping row codes to column codes. E.g., {"0": 1, "1": 3}
+
+{% if include_comment %}
+After the answer, you can put a comment explaining your choices on the next line.
+{% endif %} 

edsl/questions/templates/matrix/question_presentation.jinja

@@ -0,0 +1,20 @@
+{{question_text}}
+
+Rows:
+    {% for item in question_items %}
+{{ loop.index0 }}: {{item}}
+    {% endfor %}
+
+Columns:
+    {% for option in question_options %}
+{{ loop.index0 }}: {{option}}
+        {%- if option in option_labels %}
+ ({{option_labels[option]}})
+        {%- endif %}
+    {% endfor %}
+
+
+Select one column option for each row.
+{% if required %}
+All rows require a response.
+{% endif %} 

edsl/questions/templates/multiple_choice/answering_instructions.jinja

@@ -0,0 +1,9 @@
+{# Answering Instructions #}
+{% if use_code %}
+Respond only with the code corresponding to one of the options.
+{% else %}
+Respond only with a string corresponding to one of the options.
+{% endif %}
+{% if include_comment %}
+After the answer, you can put a comment explaining why you chose that option on the next line.
+{% endif %}

edsl/questions/templates/multiple_choice/question_presentation.jinja

@@ -0,0 +1,12 @@
+{# Question Presention #}
+{{question_text}}
+{% if use_code %}
+    {%- for option in question_options %}
+{{ loop.index0 }}: {{option}}
+    {% endfor %}
+{% else %}
+    {% for option in question_options %}
+{{option}}
+    {% endfor %}
+{% endif %}
+Only 1 option may be selected.

edsl/questions/templates/numerical/answering_instructions.jinja

@@ -0,0 +1,7 @@
+This question requires a numerical response in the form of an integer or decimal (e.g., -12, 0, 1, 2, 3.45, ...).
+Respond with just your number on a single line.
+If your response is equivalent to zero, report '0'
+
+{% if include_comment %}
+After the answer, put a comment explaining your choice on the next line.
+{% endif %}

edsl/questions/templates/numerical/question_presentation.jinja

@@ -0,0 +1,7 @@
+{{question_text}}
+{% if min_value is not none %}
+    Minimum answer value: {{min_value}}
+{% endif %}
+{% if max_value is not none %}
+    Maximum answer value: {{max_value}}
+{% endif %}

edsl/questions/templates/rank/answering_instructions.jinja

@@ -0,0 +1,11 @@
+{# Answering Instructions #}
+{% if use_code %}
+Please respond only with a comma-separated list of the code of the raked options, with square brackets. E.g., [0, 1, 3]
+{% else %}
+Please respond only with a comma-separated list of the ranked options, with square brackets. E.g., ['Good', 'Bad', 'Ugly']
+{% endif %}
+{% if include_comment %}
+After the answer, you can put a comment explaining your choice on the next line.
+{% endif %}
+
+

edsl/questions/templates/rank/question_presentation.jinja

@@ -0,0 +1,15 @@
+{{question_text}}
+{% if use_code %}
+The options are
+{% for option in question_options %}
+{{ loop.index0 }}: {{option}}
+{% endfor %}
+{% else %}
+The options are:
+{% for option in question_options %}
+{{option}}
+{% endfor %}
+{% endif %}
+{% if num_selections %}
+You can inlcude up to {{num_selections}} options in your answer.
+{% endif %}

edsl/questions/templates/top_k/answering_instructions.jinja

@@ -0,0 +1,8 @@
+{# Answering Instructions #}
+Please respond with valid JSON, formatted like so:
+{% if include_comment %}
+    {"answer": [<put comma-separated list here>], "comment": "<put explanation here>"}
+{% else %}
+    {"answer": [<put comma-separated list here>]}
+{% endif %}
+

edsl/questions/templates/top_k/question_presentation.jinja

@@ -0,0 +1,22 @@
+{{question_text}}
+{% if use_code %}
+    {% for option in question_options %}
+{{ loop.index0 }}: {{option}}
+    {% endfor %}
+{% else %}
+    {% for option in question_options %}
+{{ option }}
+    {% endfor %}    
+{% endif %}
+
+{# Restrictions #}
+{% if min_selections != None and max_selections != None and min_selections == max_selections %}
+You must select exactly {{min_selections}} options.
+{% elif min_selections != None and max_selections != None %}
+Minimum number of options that must be selected: {{min_selections}}.
+Maximum number of options that must be selected: {{max_selections}}.
+{% elif min_selections != None %}
+Minimum number of options that must be selected: {{min_selections}}.
+{% elif max_selections != None %}
+Maximum number of options that must be selected: {{max_selections}}.
+{% endif %}

edsl/questions/templates/yes_no/answering_instructions.jinja

@@ -0,0 +1,6 @@
+{# Answering Instructions #}
+Please respond with just your answer. 
+
+{% if include_comment %}
+After the answer, you can put a comment explaining your response.
+{% endif %}

edsl/questions/templates/yes_no/question_presentation.jinja

@@ -0,0 +1,12 @@
+{# Question Presention #}
+{{question_text}}
+{% if use_code %}
+    {%- for option in question_options %}
+{{ loop.index0 }}: {{option}}
+    {% endfor %}
+{% else %}
+    {% for option in question_options %}
+{{option}}
+    {% endfor %}
+{% endif %}
+Only 1 option may be selected.