edsl 0.1.50__py3-none-any.whl → 0.1.52__py3-none-any.whl

edsl/questions/question_functional.py

@@ -1,12 +1,95 @@
 from __future__ import annotations
-from typing import Optional, Callable
+from typing import Optional, Callable, Any
 import inspect
 
+from pydantic import BaseModel
+
 from .question_base import QuestionBase
+from .response_validator_abc import ResponseValidatorABC
+from .exceptions import QuestionErrors, QuestionAnswerValidationError, QuestionNotImplementedError
 
 from ..utilities.restricted_python import create_restricted_function
 
 
+class FunctionalResponse(BaseModel):
+    """
+    Pydantic model for functional question responses.
+    
+    Since functional questions are evaluated directly by Python code rather than an LLM,
+    this model primarily serves as a structured way to represent the output.
+    
+    Attributes:
+        answer: The result of the function evaluation
+        comment: Optional comment about the result
+        generated_tokens: Optional token usage data
+    
+    Examples:
+        >>> # Valid response with a numeric answer
+        >>> response = FunctionalResponse(answer=42)
+        >>> response.answer
+        42
+        
+        >>> # Valid response with a string answer and a comment
+        >>> response = FunctionalResponse(answer="Hello world", comment="Function executed successfully")
+        >>> response.answer
+        'Hello world'
+        >>> response.comment
+        'Function executed successfully'
+    """
+    answer: Any
+    comment: Optional[str] = None
+    generated_tokens: Optional[Any] = None
+
+
+class FunctionalResponseValidator(ResponseValidatorABC):
+    """
+    Validator for functional question responses.
+    
+    Since functional questions are evaluated directly and not by an LLM,
+    this validator is minimal and mainly serves for consistency with other question types.
+    """
+    required_params = []
+    valid_examples = [
+        (
+            {"answer": 42},
+            {},
+        ),
+        (
+            {"answer": "Hello world", "comment": "Function executed successfully"},
+            {},
+        ),
+    ]
+    invalid_examples = []
+    
+    def fix(self, response, verbose=False):
+        """
+        Attempt to fix an invalid response.
+        
+        Since functional questions are evaluated directly, this method is mainly
+        for consistency with other question types.
+        
+        Args:
+            response: The response to fix
+            verbose: Whether to print verbose output
+            
+        Returns:
+            The fixed response or the original response if it cannot be fixed
+        """
+        if verbose:
+            print(f"Fixing functional response: {response}")
+        
+        # Handle case where response is a raw value without the proper structure
+        if not isinstance(response, dict):
+            try:
+                return {"answer": response}
+            except Exception as e:
+                if verbose:
+                    print(f"Failed to fix response: {e}")
+                return {"answer": None, "comment": "Failed to execute function"}
+                
+        return response
+
+
 class QuestionFunctional(QuestionBase):
     """A special type of question that is *not* answered by an LLM.
 
@@ -40,7 +123,7 @@ class QuestionFunctional(QuestionBase):
     function_name = ""
 
     _response_model = None
-    response_validator_class = None
+    response_validator_class = FunctionalResponseValidator
 
     def __init__(
         self,
@@ -73,6 +156,12 @@ class QuestionFunctional(QuestionBase):
         self.question_text = question_text
         self.instructions = self.default_instructions
 
+    def create_response_model(self):
+        """
+        Returns the Pydantic model for validating responses to this question.
+        """
+        return FunctionalResponse
+
     def activate(self):
         self.activated = True
 
@@ -85,12 +174,14 @@ class QuestionFunctional(QuestionBase):
     def answer_question_directly(self, scenario, agent_traits=None):
         """Return the answer to the question, ensuring the function is activated."""
         if not self.activated:
-            raise Exception("Function not activated. Please activate it first.")
+            raise QuestionErrors("Function not activated. Please activate it first.")
         try:
-            return {"answer": self.func(scenario, agent_traits), "comment": None}
+            result = {"answer": self.func(scenario, agent_traits), "comment": None}
+            # Validate the result using the Pydantic model
+            return self.create_response_model()(**result).model_dump()
         except Exception as e:
             print("Function execution error:", e)
-            raise Exception("Error during function execution.")
+            raise QuestionErrors("Error during function execution.")
 
     def _translate_answer_code_to_answer(self, answer, scenario):
         """Required by Question, but not used by QuestionFunctional."""
@@ -98,13 +189,31 @@ class QuestionFunctional(QuestionBase):
 
     def _simulate_answer(self, human_readable=True) -> dict[str, str]:
         """Required by Question, but not used by QuestionFunctional."""
-        from .exceptions import QuestionNotImplementedError
         raise QuestionNotImplementedError("_simulate_answer not implemented for QuestionFunctional")
 
     def _validate_answer(self, answer: dict[str, str]):
-        """Required by Question, but not used by QuestionFunctional."""
-        from .exceptions import QuestionNotImplementedError
-        raise QuestionNotImplementedError("_validate_answer not implemented for QuestionFunctional")
+        """Validate the answer using the Pydantic model."""
+        try:
+            return self.create_response_model()(**answer).model_dump()
+        except Exception as e:
+            from pydantic import ValidationError
+            # Create a ValidationError with a helpful message
+            validation_error = ValidationError.from_exception_data(
+                title='FunctionalResponse',
+                line_errors=[{
+                    'type': 'value_error',
+                    'loc': ('answer',),
+                    'msg': f'Function response validation failed: {str(e)}',
+                    'input': answer,
+                    'ctx': {'error': str(e)}
+                }]
+            )
+            raise QuestionAnswerValidationError(
+                message=f"Invalid function response: {str(e)}",
+                data=answer,
+                model=self.create_response_model(),
+                pydantic_error=validation_error
+            )
 
     @property
     def question_html_content(self) -> str:

edsl/questions/{derived/question_likert_five.py → question_likert_five.py}

@@ -1,7 +1,7 @@
 from __future__ import annotations
 from typing import Optional
-from ..question_multiple_choice import QuestionMultipleChoice
-from ..decorators import inject_exception
+from .question_multiple_choice import QuestionMultipleChoice
+from .decorators import inject_exception
 
 
 class QuestionLikertFive(QuestionMultipleChoice):

edsl/questions/{derived/question_linear_scale.py → question_linear_scale.py}

@@ -1,10 +1,9 @@
 from __future__ import annotations
 from typing import Optional
 
-from ..descriptors import QuestionOptionsDescriptor, OptionLabelDescriptor
-from ..question_multiple_choice import QuestionMultipleChoice
-
-from ..decorators import inject_exception
+from .descriptors import QuestionOptionsDescriptor, OptionLabelDescriptor
+from .question_multiple_choice import QuestionMultipleChoice
+from .decorators import inject_exception
 
 
 class QuestionLinearScale(QuestionMultipleChoice):

edsl/questions/question_list.py

@@ -2,9 +2,8 @@ from __future__ import annotations
 import json
 from typing import Any, Optional, Union, ForwardRef
 
-from pydantic import Field
+from pydantic import Field, model_validator, ValidationError
 from json_repair import repair_json
-from .exceptions import QuestionAnswerValidationError
 from .question_base import QuestionBase
 from .descriptors import IntegerOrNoneDescriptor
 from .decorators import inject_exception
@@ -60,11 +59,44 @@ def create_model(min_list_items: Optional[int], max_list_items: Optional[int], p
     from pydantic import BaseModel
 
     if permissive or (max_list_items is None and min_list_items is None):
-
         class ListResponse(BaseModel):
+            """
+            Pydantic model for validating list responses with no constraints.
+            
+            Examples:
+                >>> # Valid response with any number of items
+                >>> response = ListResponse(answer=["one", "two", "three"])
+                >>> response.answer
+                ['one', 'two', 'three']
+                
+                >>> # Empty list is valid in permissive mode
+                >>> response = ListResponse(answer=[])
+                >>> response.answer
+                []
+                
+                >>> # Missing answer field raises error
+                >>> try:
+                ...     ListResponse(you="will never be able to do this!")
+                ... except Exception as e:
+                ...     "Field required" in str(e)
+                True
+            """
             answer: list[Any]
             comment: Optional[str] = None
             generated_tokens: Optional[str] = None
+            
+            @classmethod
+            def model_validate(cls, obj, *args, **kwargs):
+                try:
+                    return super().model_validate(obj, *args, **kwargs)
+                except ValidationError as e:
+                    from .exceptions import QuestionAnswerValidationError
+                    raise QuestionAnswerValidationError(
+                        message=f"Invalid list response: {e}",
+                        data=obj,
+                        model=cls,
+                        pydantic_error=e
+                    )
 
     else:
         # Determine field constraints
@@ -78,58 +110,246 @@ def create_model(min_list_items: Optional[int], max_list_items: Optional[int], p
 
         class ListResponse(BaseModel):
             """
-            >>> nr = ListResponse(answer=["Apple", "Cherry"])
-            >>> nr.dict()
-            {'answer': ['Apple', 'Cherry'], 'comment': None, 'generated_tokens': None}
+            Pydantic model for validating list responses with size constraints.
+            
+            Examples:
+                >>> # Create a model with min=2, max=4 items
+                >>> ConstrainedList = create_model(min_list_items=2, max_list_items=4, permissive=False)
+                
+                >>> # Valid response within constraints
+                >>> response = ConstrainedList(answer=["Apple", "Cherry", "Banana"])
+                >>> len(response.answer)
+                3
+                
+                >>> # Too few items raises error
+                >>> try:
+                ...     ConstrainedList(answer=["Apple"])
+                ... except QuestionAnswerValidationError as e:
+                ...     "must have at least 2 items" in str(e)
+                True
+                
+                >>> # Too many items raises error
+                >>> try:
+                ...     ConstrainedList(answer=["A", "B", "C", "D", "E"])
+                ... except QuestionAnswerValidationError as e:
+                ...     "cannot have more than 4 items" in str(e)
+                True
+                
+                >>> # Optional comment is allowed
+                >>> response = ConstrainedList(
+                ...     answer=["Apple", "Cherry"],
+                ...     comment="These are my favorites"
+                ... )
+                >>> response.comment
+                'These are my favorites'
+                
+                >>> # Generated tokens are optional
+                >>> response = ConstrainedList(
+                ...     answer=["Apple", "Cherry"],
+                ...     generated_tokens="Apple, Cherry"
+                ... )
+                >>> response.generated_tokens
+                'Apple, Cherry'
             """
 
             answer: list[Any] = Field(**field_kwargs)
             comment: Optional[str] = None
             generated_tokens: Optional[str] = None
 
+            @model_validator(mode='after')
+            def validate_list_constraints(self):
+                """
+                Validate that the list meets size constraints.
+                
+                Returns:
+                    The validated model instance.
+                    
+                Raises:
+                    QuestionAnswerValidationError: If list size constraints are violated.
+                """
+                if max_list_items is not None and len(self.answer) > max_list_items:
+                    from .exceptions import QuestionAnswerValidationError
+                    validation_error = ValidationError.from_exception_data(
+                        title='ListResponse',
+                        line_errors=[{
+                            'type': 'value_error',
+                            'loc': ('answer',),
+                            'msg': f'List cannot have more than {max_list_items} items',
+                            'input': self.answer,
+                            'ctx': {'error': 'Too many items'}
+                        }]
+                    )
+                    raise QuestionAnswerValidationError(
+                        message=f"List cannot have more than {max_list_items} items",
+                        data=self.model_dump(),
+                        model=self.__class__,
+                        pydantic_error=validation_error
+                    )
+                
+                if min_list_items is not None and len(self.answer) < min_list_items:
+                    from .exceptions import QuestionAnswerValidationError
+                    validation_error = ValidationError.from_exception_data(
+                        title='ListResponse',
+                        line_errors=[{
+                            'type': 'value_error',
+                            'loc': ('answer',),
+                            'msg': f'List must have at least {min_list_items} items',
+                            'input': self.answer,
+                            'ctx': {'error': 'Too few items'}
+                        }]
+                    )
+                    raise QuestionAnswerValidationError(
+                        message=f"List must have at least {min_list_items} items",
+                        data=self.model_dump(),
+                        model=self.__class__,
+                        pydantic_error=validation_error
+                    )
+                return self
+                
+            @classmethod
+            def model_validate(cls, obj, *args, **kwargs):
+                try:
+                    return super().model_validate(obj, *args, **kwargs)
+                except ValidationError as e:
+                    from .exceptions import QuestionAnswerValidationError
+                    raise QuestionAnswerValidationError(
+                        message=f"Invalid list response: {e}",
+                        data=obj,
+                        model=cls,
+                        pydantic_error=e
+                    )
+
     return ListResponse
 
 
 class ListResponseValidator(ResponseValidatorABC):
     required_params = ["min_list_items", "max_list_items", "permissive"]
     valid_examples = [({"answer": ["hello", "world"]}, {"max_list_items": 5})]
-
     invalid_examples = [
         (
             {"answer": ["hello", "world", "this", "is", "a", "test"]},
             {"max_list_items": 5},
-            "Too many items.",
+            "List cannot have more than 5 items",
         ),
         (
             {"answer": ["hello"]},
             {"min_list_items": 2},
-            "Too few items.",
+            "List must have at least 2 items",
         ),
     ]
+    
+    def validate(
+        self,
+        raw_edsl_answer_dict: dict,
+        fix=False,
+        verbose=False,
+        replacement_dict: dict = None,
+    ) -> dict:
+        """Override validate to handle missing answer key properly."""
+        # Check for missing answer key
+        if "answer" not in raw_edsl_answer_dict:
+            from .exceptions import QuestionAnswerValidationError
+            from pydantic import ValidationError
+            
+            # Create a synthetic validation error
+            validation_error = ValidationError.from_exception_data(
+                title='ListResponse',
+                line_errors=[{
+                    'type': 'missing',
+                    'loc': ('answer',),
+                    'msg': 'Field required',
+                    'input': raw_edsl_answer_dict,
+                }]
+            )
+            
+            raise QuestionAnswerValidationError(
+                message="Missing required 'answer' field in response",
+                data=raw_edsl_answer_dict,
+                model=self.response_model,
+                pydantic_error=validation_error
+            )
+        
+        # Check if answer is not a list
+        if "answer" in raw_edsl_answer_dict and not isinstance(raw_edsl_answer_dict["answer"], list):
+            from .exceptions import QuestionAnswerValidationError
+            from pydantic import ValidationError
+            
+            # Create a synthetic validation error
+            validation_error = ValidationError.from_exception_data(
+                title='ListResponse',
+                line_errors=[{
+                    'type': 'list_type',
+                    'loc': ('answer',),
+                    'msg': 'Input should be a valid list',
+                    'input': raw_edsl_answer_dict["answer"],
+                }]
+            )
+            
+            raise QuestionAnswerValidationError(
+                message=f"Answer must be a list (got {type(raw_edsl_answer_dict['answer']).__name__})",
+                data=raw_edsl_answer_dict,
+                model=self.response_model,
+                pydantic_error=validation_error
+            )
+        
+        # Continue with parent validation
+        return super().validate(raw_edsl_answer_dict, fix, verbose, replacement_dict)
 
     def _check_constraints(self, response) -> None:
-        if (
-            self.max_list_items is not None
-            and len(response.answer) > self.max_list_items
-        ):
-            raise QuestionAnswerValidationError("Too many items.")
-            
-        if (
-            self.min_list_items is not None
-            and len(response.answer) < self.min_list_items
-        ):
-            raise QuestionAnswerValidationError("Too few items.")
+        # This method can now be removed since validation is handled in the Pydantic model
+        pass
 
     def fix(self, response, verbose=False):
+        """
+        Fix common issues in list responses by splitting strings into lists.
+        
+        Examples:
+            >>> from edsl import QuestionList
+            >>> q = QuestionList.example(min_list_items=2, max_list_items=4)
+            >>> validator = q.response_validator
+            
+            >>> # Fix a string that should be a list
+            >>> bad_response = {"answer": "apple,banana,cherry"}
+            >>> try:
+            ...     validator.validate(bad_response)
+            ... except Exception:
+            ...     fixed = validator.fix(bad_response)
+            ...     validated = validator.validate(fixed)
+            ...     validated  # Show full response
+            {'answer': ['apple', 'banana', 'cherry'], 'comment': None, 'generated_tokens': None}
+
+            >>> # Fix using generated_tokens when answer is invalid
+            >>> bad_response = {
+            ...     "answer": None,
+            ...     "generated_tokens": "pizza, pasta, salad"
+            ... }
+            >>> try:
+            ...     validator.validate(bad_response)
+            ... except Exception:
+            ...     fixed = validator.fix(bad_response)
+            ...     validated = validator.validate(fixed)
+            ...     validated
+            {'answer': ['pizza', ' pasta', ' salad'], 'comment': None, 'generated_tokens': None}
+
+            >>> # Preserve comments during fixing
+            >>> bad_response = {
+            ...     "answer": "red,blue,green",
+            ...     "comment": "These are colors"
+            ... }
+            >>> fixed = validator.fix(bad_response)
+            >>> fixed == {
+            ...     "answer": ["red", "blue", "green"],
+            ...     "comment": "These are colors"
+            ... }
+            True
+        """
         if verbose:
             print(f"Fixing list response: {response}")
         answer = str(response.get("answer") or response.get("generated_tokens", ""))
-        if len(answer.split(",")) > 0:
-            return (
-                {"answer": answer.split(",")} | {"comment": response.get("comment")}
-                if "comment" in response
-                else {}
-            )
+        result = {"answer": answer.split(",")}
+        if "comment" in response:
+            result["comment"] = response["comment"]
+        return result
 
     def _post_process(self, edsl_answer_dict):
         edsl_answer_dict["answer"] = [