edsl 0.1.32__py3-none-any.whl → 0.1.33__py3-none-any.whl

edsl/questions/ResponseValidatorABC.py

@@ -0,0 +1,170 @@
+from abc import ABC, abstractmethod
+from pydantic import BaseModel, Field, field_validator
+
+# from decimal import Decimal
+from typing import Optional, Any, List, TypedDict
+
+from edsl.exceptions import QuestionAnswerValidationError
+from pydantic import ValidationError
+
+
+class BaseResponse(BaseModel):
+    answer: Any
+    comment: Optional[str] = None
+    generated_tokens: Optional[str] = None
+
+
+class ResponseValidatorABC(ABC):
+    required_params: List[str] = []
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        required_class_vars = ["required_params", "valid_examples", "invalid_examples"]
+        for var in required_class_vars:
+            if not hasattr(cls, var):
+                raise ValueError(f"Class {cls.__name__} must have a '{var}' attribute.")
+
+    def __init__(
+        self,
+        response_model: type[BaseModel],
+        exception_to_throw: Optional[Exception] = None,
+        override_answer: Optional[dict] = None,
+        **kwargs,
+    ):
+        self.response_model = response_model
+        self.exception_to_throw = exception_to_throw  # for testing
+        self.override_answer = override_answer  # for testing
+        self.original_exception = None
+
+        # Validate required parameters
+        missing_params = [
+            param for param in self.required_params if param not in kwargs
+        ]
+        if missing_params:
+            raise ValueError(
+                f"Missing required parameters: {', '.join(missing_params)}"
+            )
+
+        # Set attributes
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+        if not hasattr(self, "permissive"):
+            self.permissive = False
+
+        self.fixes_tried = 0
+
+    class RawEdslAnswerDict(TypedDict):
+        answer: Any
+        comment: Optional[str]
+        generated_tokens: Optional[str]
+
+    def _preprocess(self, data: RawEdslAnswerDict) -> RawEdslAnswerDict:
+        """This is for testing purposes. A question can be given an exception to throw or an answer to always return.
+
+        >>> rv = ResponseValidatorABC.example()
+        >>> rv.override_answer = {"answer": 42}
+        >>> rv.validate({"answer": 23})
+        {'answer': 42, 'comment': None, 'generated_tokens': None}
+        """
+        if self.exception_to_throw:
+            raise self.exception_to_throw
+        return self.override_answer if self.override_answer else data
+
+    def _base_validate(self, data: RawEdslAnswerDict) -> BaseModel:
+        """This is the main validation function. It takes the response_model and checks the data against it, returning the instantiated model.
+
+        >>> rv = ResponseValidatorABC.example("numerical")
+        >>> rv._base_validate({"answer": 42})
+        ConstrainedNumericResponse(answer=42, comment=None, generated_tokens=None)
+        """
+        try:
+            return self.response_model(**data)
+        except ValidationError as e:
+            raise QuestionAnswerValidationError(e, data=data, model=self.response_model)
+
+    def post_validation_answer_convert(self, data):
+        return data
+
+    class EdslAnswerDict(TypedDict):
+        answer: Any
+        comment: Optional[str]
+        generated_tokens: Optional[str]
+
+    def validate(
+        self, raw_edsl_answer_dict: RawEdslAnswerDict, fix=False, verbose=False
+    ) -> EdslAnswerDict:
+        """This is the main validation function.
+
+        >>> rv = ResponseValidatorABC.example("numerical")
+        >>> rv.validate({"answer": 42})
+        {'answer': 42, 'comment': None, 'generated_tokens': None}
+        >>> rv.max_value
+        86.7
+        >>> rv.validate({"answer": "120"})
+        Traceback (most recent call last):
+        ...
+        edsl.exceptions.questions.QuestionAnswerValidationError:...
+        >>> from edsl import QuestionNumerical
+        >>> q = QuestionNumerical.example()
+        >>> q.permissive = True
+        >>> rv = q.response_validator
+        >>> rv.validate({"answer": "120"})
+        {'answer': 120, 'comment': None, 'generated_tokens': None}
+        >>> rv.validate({"answer": "poo"})
+        Traceback (most recent call last):
+        ...
+        edsl.exceptions.questions.QuestionAnswerValidationError:...
+        """
+        proposed_edsl_answer_dict = self._preprocess(raw_edsl_answer_dict)
+        try:
+            pydantic_edsl_answer: BaseModel = self._base_validate(
+                proposed_edsl_answer_dict
+            )
+            edsl_answer_dict = self._extract_answer(pydantic_edsl_answer)
+            return self._post_process(edsl_answer_dict)
+        except QuestionAnswerValidationError as e:
+            if verbose:
+                print(f"Failed to validate {raw_edsl_answer_dict}; {str(e)}")
+            return self._handle_exception(e, raw_edsl_answer_dict)
+
+    def _handle_exception(self, e: Exception, raw_edsl_answer_dict) -> EdslAnswerDict:
+        if self.fixes_tried == 0:
+            self.original_exception = e
+
+        if self.fixes_tried == 0 and hasattr(self, "fix"):
+            self.fixes_tried += 1
+            fixed_data = self.fix(raw_edsl_answer_dict)
+            try:
+                return self.validate(fixed_data, fix=True)
+            except Exception as e:
+                pass  # we don't log failed fixes
+
+        raise QuestionAnswerValidationError(
+            self.original_exception,
+            data=raw_edsl_answer_dict,
+            model=self.response_model,
+        )
+
+    def _check_constraints(self, pydantic_edsl_answer: BaseModel) -> dict:
+        pass
+
+    def _extract_answer(self, response: BaseModel) -> EdslAnswerDict:
+        return response.model_dump()
+
+    def _post_process(self, edsl_answer_dict: EdslAnswerDict) -> EdslAnswerDict:
+        return edsl_answer_dict
+
+    @classmethod
+    def example(cls, question_type="numerical"):
+        from edsl import Question
+
+        q = Question.example(question_type)
+        return q.response_validator
+
+
+# Example usage
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/questions/__init__.py

@@ -6,22 +6,21 @@ from edsl.questions.RegisterQuestionsMeta import RegisterQuestionsMeta
 from edsl.questions.QuestionBase import QuestionBase
 
 # Core Questions
-from edsl.questions.QuestionBudget import QuestionBudget
 from edsl.questions.QuestionCheckBox import QuestionCheckBox
 from edsl.questions.QuestionExtract import QuestionExtract
 from edsl.questions.QuestionFreeText import QuestionFreeText
-
 from edsl.questions.QuestionFunctional import QuestionFunctional
 from edsl.questions.QuestionList import QuestionList
 from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
 from edsl.questions.QuestionNumerical import QuestionNumerical
+from edsl.questions.QuestionBudget import QuestionBudget
 from edsl.questions.QuestionRank import QuestionRank
 
-# # Questions derived from core questions
+# # # Questions derived from core questions
 from edsl.questions.derived.QuestionLikertFive import QuestionLikertFive
 from edsl.questions.derived.QuestionLinearScale import QuestionLinearScale
-from edsl.questions.derived.QuestionTopK import QuestionTopK
 from edsl.questions.derived.QuestionYesNo import QuestionYesNo
+from edsl.questions.derived.QuestionTopK import QuestionTopK
 
 # # Compose Questions
 # from edsl.questions.compose_questions import compose_questions

edsl/questions/decorators.py

@@ -0,0 +1,21 @@
+from typing import Optional, Callable, TypeVar
+
+T = TypeVar("T")
+
+
+def inject_exception(func: Callable[..., T]) -> Callable[..., T]:
+    def wrapper(
+        cls,
+        exception_to_throw: Optional[Exception] = None,
+        override_answer: Optional[dict] = None,
+        *args,
+        **kwargs
+    ) -> T:
+        base_instance = func(cls, *args, **kwargs)
+        if exception_to_throw:
+            base_instance.exception_to_throw = exception_to_throw
+        if override_answer:
+            base_instance.override_answer = override_answer
+        return base_instance
+
+    return wrapper

edsl/questions/derived/QuestionLikertFive.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 from typing import Optional
 from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
 
+from edsl.questions.decorators import inject_exception
+
 
 class QuestionLikertFive(QuestionMultipleChoice):
     """This question prompts the agent to respond to a statement on a 5-point Likert scale."""
@@ -14,31 +16,34 @@ class QuestionLikertFive(QuestionMultipleChoice):
         "Agree",
         "Strongly agree",
     ]
-    # default_instructions = QuestionMultipleChoice.default_instructions
 
     def __init__(
         self,
         question_name: str,
         question_text: str,
         question_options: Optional[list[str]] = likert_options,
+        answering_instructions: Optional[str] = None,
+        question_presentation: Optional[str] = None,
+        include_comment: bool = True,
     ):
         """Initialize the question.
 
         :param question_name: The name of the question.
         :param question_text: The text of the question.
         :param question_options: The options the respondent should select from (list of strings). If not provided, the default Likert options are used (['Strongly disagree', 'Disagree', 'Neutral', 'Agree', 'Strongly agree']). To view them, run `QuestionLikertFive.likert_options`.
-        :param instructions: Instructions for the question. If not provided, the default instructions are used. To view them, run `QuestionLikertFive.default_instructions`.
         """
         super().__init__(
             question_name=question_name,
             question_text=question_text,
             question_options=question_options,
+            use_code=False,
+            include_comment=include_comment,
+            answering_instructions=answering_instructions,
+            question_presentation=question_presentation,
         )
 
-    ################
-    # Helpful
-    ################
     @classmethod
+    @inject_exception
     def example(cls) -> QuestionLikertFive:
         """Return an example question."""
         return cls(

edsl/questions/derived/QuestionLinearScale.py

@@ -4,6 +4,8 @@ from typing import Optional
 from edsl.questions.descriptors import QuestionOptionsDescriptor, OptionLabelDescriptor
 from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
 
+from edsl.questions.decorators import inject_exception
+
 
 class QuestionLinearScale(QuestionMultipleChoice):
     """This question prompts the agent to respond to a statement on a linear scale."""
@@ -18,6 +20,9 @@ class QuestionLinearScale(QuestionMultipleChoice):
         question_text: str,
         question_options: list[int],
         option_labels: Optional[dict[int, str]] = None,
+        answering_instructions: Optional[str] = None,
+        question_presentation: Optional[str] = None,
+        include_comment: Optional[bool] = True,
     ):
         """Instantiate a new QuestionLinearScale.
 
@@ -31,21 +36,29 @@ class QuestionLinearScale(QuestionMultipleChoice):
             question_name=question_name,
             question_text=question_text,
             question_options=question_options,
+            use_code=False,  # question linear scale will have it's own code
+            include_comment=include_comment,
         )
         self.question_options = question_options
-        self.option_labels = option_labels
+        self.option_labels = (
+            {int(k): v for k, v in option_labels.items()} if option_labels else {}
+        )
+        self.answering_instructions = answering_instructions
+        self.question_presentation = question_presentation
 
     ################
     # Helpful
     ################
     @classmethod
-    def example(cls) -> QuestionLinearScale:
+    @inject_exception
+    def example(cls, include_comment: bool = True) -> QuestionLinearScale:
         """Return an example of a linear scale question."""
         return cls(
             question_text="How much do you like ice cream?",
             question_options=[1, 2, 3, 4, 5],
             question_name="ice_cream",
             option_labels={1: "I hate it", 5: "I love it"},
+            include_comment=include_comment,
         )
 
 

edsl/questions/derived/QuestionTopK.py

@@ -3,6 +3,7 @@ from typing import Optional
 
 from edsl.exceptions import QuestionCreationValidationError
 from edsl.questions.QuestionCheckBox import QuestionCheckBox
+from edsl.questions.decorators import inject_exception
 
 
 class QuestionTopK(QuestionCheckBox):
@@ -17,6 +18,9 @@ class QuestionTopK(QuestionCheckBox):
         question_options: list[str],
         min_selections: int,
         max_selections: int,
+        question_presentation: Optional[str] = None,
+        answering_instructions: Optional[str] = None,
+        include_comment: Optional[bool] = True,
     ):
         """Initialize the question.
 
@@ -32,6 +36,9 @@ class QuestionTopK(QuestionCheckBox):
             question_options=question_options,
             min_selections=min_selections,
             max_selections=max_selections,
+            question_presentation=question_presentation,
+            answering_instructions=answering_instructions,
+            include_comment=include_comment,
         )
         if min_selections != max_selections:
             raise QuestionCreationValidationError(
@@ -46,7 +53,8 @@ class QuestionTopK(QuestionCheckBox):
     # Helpful
     ################
     @classmethod
-    def example(cls) -> QuestionTopK:
+    @inject_exception
+    def example(cls, include_comment: bool = True) -> QuestionTopK:
         """Return an example question."""
         return cls(
             question_name="two_fruits",
@@ -54,6 +62,7 @@ class QuestionTopK(QuestionCheckBox):
             question_options=["apple", "banana", "carrot", "durian"],
             min_selections=2,
             max_selections=2,
+            include_comment=include_comment,
         )
 
 

edsl/questions/derived/QuestionYesNo.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
+from typing import Optional
 from edsl.questions.descriptors import QuestionOptionsDescriptor
 from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
 
+from edsl.questions.decorators import inject_exception
+
 
 class QuestionYesNo(QuestionMultipleChoice):
     """This question prompts the agent to respond with 'Yes' or 'No'."""
@@ -13,7 +16,10 @@ class QuestionYesNo(QuestionMultipleChoice):
         self,
         question_name: str,
         question_text: str,
-        question_options: list[str] = ["Yes", "No"],
+        question_options: list[str] = ["No", "Yes"],
+        answering_instructions: Optional[str] = None,
+        question_presentation: Optional[str] = None,
+        include_comment: Optional[bool] = True,
     ):
         """Instantiate a new QuestionYesNo.
 
@@ -25,6 +31,10 @@ class QuestionYesNo(QuestionMultipleChoice):
             question_name=question_name,
             question_text=question_text,
             question_options=question_options,
+            use_code=False,
+            answering_instructions=answering_instructions,
+            question_presentation=question_presentation,
+            include_comment=include_comment,
         )
         self.question_options = question_options
 
@@ -32,9 +42,14 @@ class QuestionYesNo(QuestionMultipleChoice):
     # Helpful
     ################
     @classmethod
-    def example(cls) -> QuestionYesNo:
+    @inject_exception
+    def example(cls, include_comment: bool = True) -> QuestionYesNo:
         """Return an example of a yes/no question."""
-        return cls(question_name="is_it_equal", question_text="Is 5 + 5 equal to 11?")
+        return cls(
+            question_name="is_it_equal",
+            question_text="Is 5 + 5 equal to 11?",
+            include_comment=include_comment,
+        )
 
 
 def main():
@@ -59,3 +74,9 @@ def main():
     import doctest
 
     doctest.testmod(optionflags=doctest.ELLIPSIS)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/questions/descriptors.py

@@ -206,12 +206,14 @@ class OptionLabelDescriptor(BaseDescriptor):
 
     def validate(self, value, instance):
         """Validate the value is a string."""
-        if value is not None:
-            if min(value.keys()) != min(instance.question_options):
+        # key_values = [int(v) for v in value.keys()]
+
+        if value and (key_values := [float(v) for v in value.keys()]) != []:
+            if min(key_values) != min(instance.question_options):
                 raise QuestionCreationValidationError(
                     f"First option needs a label (got {value})"
                 )
-            if max(value.keys()) != max(instance.question_options):
+            if max(key_values) != max(instance.question_options):
                 raise QuestionCreationValidationError(
                     f"Last option needs a label (got {value})"
                 )
@@ -219,12 +221,17 @@ class OptionLabelDescriptor(BaseDescriptor):
                 raise QuestionCreationValidationError(
                     "Option labels must be strings (got {value})."
                 )
-            for key in value.keys():
+            for key in key_values:
                 if key not in instance.question_options:
                     raise QuestionCreationValidationError(
                         f"Option label key ({key}) is not in question options ({instance.question_options})."
                     )
 
+            if len(value.values()) != len(set(value.values())):
+                raise QuestionCreationValidationError(
+                    f"Option labels must be unique (got {value})."
+                )
+
 
 class QuestionNameDescriptor(BaseDescriptor):
     """Validate that the `question_name` attribute is a valid variable name."""
@@ -233,6 +240,15 @@ class QuestionNameDescriptor(BaseDescriptor):
         """Validate the value is a valid variable name."""
         from edsl.utilities.utilities import is_valid_variable_name
 
+        if "{{" in value and "}}" in value:
+            # they're trying to use a dynamic question name - let's let this play out
+            return None
+
+        if value.endswith("_comment") or value.endswith("_generated_tokens"):
+            raise QuestionCreationValidationError(
+                f"`question_name` cannot end with '_comment' or '_generated_tokens - (got {value})."
+            )
+
         if not is_valid_variable_name(value):
             raise QuestionCreationValidationError(
                 f"`question_name` is not a valid variable name (got {value})."
@@ -279,7 +295,7 @@ class QuestionOptionsDescriptor(BaseDescriptor):
         >>> _ = q_class("dynamic_options")
         Traceback (most recent call last):
         ...
-        edsl.exceptions.questions.QuestionCreationValidationError: Dynamic question options must be of the form: '{{ question_options }}'.
+        edsl.exceptions.questions.QuestionCreationValidationError: ...
         """
         if isinstance(value, str):
             # Check if the string is a dynamic question option
@@ -287,7 +303,7 @@ class QuestionOptionsDescriptor(BaseDescriptor):
                 return None
             else:
                 raise QuestionCreationValidationError(
-                    "Dynamic question options must be of the form: '{{ question_options }}'."
+                    f"Dynamic question options must have jina2 braces - instead received: {value}."
                 )
         if not isinstance(value, list):
             raise QuestionCreationValidationError(
@@ -356,7 +372,21 @@ class QuestionOptionsDescriptor(BaseDescriptor):
 
 
 class QuestionTextDescriptor(BaseDescriptor):
-    """Validate that the `question_text` attribute is a string."""
+    """Validate that the `question_text` attribute is a string.
+
+
+    >>> class TestQuestion:
+    ...     question_text = QuestionTextDescriptor()
+    ...     def __init__(self, question_text: str):
+    ...         self.question_text = question_text
+
+    >>> _ = TestQuestion("What is the capital of France?")
+    >>> _ = TestQuestion("What is the capital of France? {{variable}}")
+    >>> _ = TestQuestion("What is the capital of France? {{variable name}}")
+    Traceback (most recent call last):
+    ...
+    edsl.exceptions.questions.QuestionCreationValidationError: Question text contains an invalid identifier: 'variable name'
+    """
 
     def validate(self, value, instance):
         """Validate the value is a string."""
@@ -373,6 +403,12 @@ class QuestionTextDescriptor(BaseDescriptor):
                 f"WARNING: Question text contains a single-braced substring: If you intended to parameterize the question with a Scenario this should be changed to a double-braced substring, e.g. {{variable}}.\nSee details on constructing Scenarios in the docs: https://docs.expectedparrot.com/en/latest/scenarios.html",
                 UserWarning,
             )
+        # iterate through all doubles braces and check if they are valid python identifiers
+        for match in re.finditer(r"\{\{([^\{\}]+)\}\}", value):
+            if " " in match.group(1).strip():
+                raise QuestionCreationValidationError(
+                    f"Question text contains an invalid identifier: '{match.group(1)}'"
+                )
 
 
 if __name__ == "__main__":

edsl/questions/prompt_templates/question_budget.jinja

@@ -0,0 +1,13 @@
+You are being asked the following question: {{question_text}}
+The options are:
+{% for option in question_options %}
+    {{ loop.index0 }}: {{option}}
+{% endfor %}                       
+Return a valid JSON formatted as follows, with a dictionary for your "answer" 
+where the keys are the option numbers and the values are the amounts you want 
+to allocate to the options, and the sum of the values is {{budget_sum}}:
+
+{"answer": {<put dict of option numbers and allocation amounts here>}, "comment": "<put explanation here>"}
+Example response for a budget of 100 and 4 options: 
+{"answer": {"0": 25, "1": 25, "2": 25, "3": 25}, "comment": "I allocated 25 to each option."}
+There must be an allocation listed for each item (including 0).

edsl/questions/prompt_templates/question_checkbox.jinja

@@ -0,0 +1,32 @@
+{# Question Presention #}
+{{question_text}}
+{% if use_code %}
+    {% for option in question_options %}
+        {{ loop.index0 }}: {{option}}
+    {% endfor %}
+{% else %}
+    {% for option in question_options %}
+        {{ option }}
+    {% endfor %}    
+{% endif %}
+
+{# Restrictions #}
+{% if min_selections != None and max_selections != None and min_selections == max_selections %}
+    You must select exactly {{min_selections}} options.
+{% elif min_selections != None and max_selections != None %}
+    Minimum number of options that must be selected: {{min_selections}}.
+    Maximum number of options that must be selected: {{max_selections}}.
+{% elif min_selections != None %}
+    Minimum number of options that must be selected: {{min_selections}}.
+{% elif max_selections != None %}
+    Maximum number of options that must be selected: {{max_selections}}.
+{% endif %}
+
+{# Answering Instructions #}
+Please respond with valid JSON, formatted like so:
+{% if include_comment %}
+    {"answer": [<put comma-separated list here>], "comment": "<put explanation here>"}
+{% else %}
+    {"answer": [<put comma-separated list here>]}
+{% endif %}
+

edsl/questions/prompt_templates/question_extract.jinja

@@ -0,0 +1,11 @@
+{{question_text}}
+
+Create an ANSWER should be formatted like this: 
+{{ answer_template }}
+
+It should have the same keys but values extracted from the input.
+If the value of a key is not present in the input, fill with "null".
+
+Return a valid JSON formatted like this:
+{"answer": <put your ANSWER here>}
+ONLY RETURN THE JSON, AND NOTHING ELSE.

edsl/questions/prompt_templates/question_free_text.jinja

@@ -0,0 +1,3 @@
+{{question_text}}
+Return a valid JSON formatted like this: 
+{"answer": "<put free text answer here>"}

edsl/questions/prompt_templates/question_linear_scale.jinja

@@ -0,0 +1,11 @@
+    {{question_text}}
+        {% for option in question_options %}
+            {{option}} : {{ option_labels.get(option, "") }}
+        {% endfor %}
+    Return a valid JSON formatted like this, selecting only the code of the option (codes start at 0):
+    {% if include_comment %}  
+        {"answer": <put answer code here>, "comment": <comment>} 
+    {% else %} 
+        {"answer": <put answer here>} 
+    {% endif %}
+    Only 1 option may be selected.

edsl/questions/prompt_templates/question_list.jinja

@@ -0,0 +1,17 @@
+{{question_text}}
+
+Your response should be only a valid JSON in the following format:
+{% if include_comment %}
+    {
+    "answer": [<comma-separated list of responsive words or phrases as independent strings>],
+    "comment": "<put comment here>"
+    }
+{% else %}
+    {
+    "answer": [<comma-separated list of responsive words or phrases as independent strings>],
+    }
+{% endif %}
+
+{% if max_list_items is not none %}
+The list must not contain more than {{ max_list_items }} items.
+{% endif %}

edsl/questions/prompt_templates/question_multiple_choice.jinja

@@ -0,0 +1,33 @@
+{# Question Presention #}
+{{question_text}}
+
+{% if use_code %}
+    {% for option in question_options %}
+        {{ loop.index0 }}: {{option}}
+    {% endfor %}
+{% else %}
+    {% for option in question_options %}
+        {{option}}
+    {% endfor %}
+{% endif %}
+
+Only 1 option may be selected.
+
+{# Answering Instructions #}
+Return a valid JSON formatted like this:
+
+{% if use_code %}
+    {% if include_comment %}
+        {"answer": <put answer code here>, "comment": "<put explanation here>"}
+    {% else %}
+        {"answer": <put answer code here>}
+    {% endif %}
+{% else %}
+
+    {% if include_comment %}
+        {"answer": <text of option>, "comment": "<put explanation here>"}
+    {% else %}
+        {"answer": <put option here>}
+    {% endif %}
+
+{% endif %}