edsl 0.1.39.dev1__py3-none-any.whl → 0.1.39.dev2__py3-none-any.whl

edsl/questions/QuestionMatrix.py

@@ -0,0 +1,265 @@
+from __future__ import annotations
+from typing import Union, Optional, Dict, List, Any
+
+from pydantic import BaseModel, Field, field_validator
+from jinja2 import Template
+import random
+from edsl.questions.QuestionBase import QuestionBase
+from edsl.questions.descriptors import (
+    QuestionOptionsDescriptor,
+    OptionLabelDescriptor,
+    QuestionTextDescriptor,
+)
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.decorators import inject_exception
+from edsl.exceptions.questions import (
+    QuestionAnswerValidationError,
+    QuestionCreationValidationError,
+)
+
+
+def create_matrix_response(
+    question_items: List[str],
+    question_options: List[Union[int, str, float]],
+    permissive: bool = False,
+):
+    """Create a response model for matrix questions.
+
+    The response model validates that:
+    1. All question items are answered
+    2. Each answer is from the allowed options
+    """
+
+    if permissive:
+
+        class MatrixResponse(BaseModel):
+            answer: Dict[str, Any]
+            comment: Optional[str] = None
+            generated_tokens: Optional[Any] = None
+
+    else:
+
+        class MatrixResponse(BaseModel):
+            answer: Dict[str, Union[int, str, float]] = Field(
+                ..., description="Mapping of items to selected options"
+            )
+            comment: Optional[str] = None
+            generated_tokens: Optional[Any] = None
+
+            @field_validator("answer")
+            def validate_answer(cls, v, values, **kwargs):
+                # Check that all items have responses
+                if not all(item in v for item in question_items):
+                    missing = set(question_items) - set(v.keys())
+                    raise ValueError(f"Missing responses for items: {missing}")
+
+                # Check that all responses are valid options
+                if not all(answer in question_options for answer in v.values()):
+                    invalid = [ans for ans in v.values() if ans not in question_options]
+                    raise ValueError(f"Invalid options selected: {invalid}")
+                return v
+
+    return MatrixResponse
+
+
+class MatrixResponseValidator(ResponseValidatorABC):
+    required_params = ["question_items", "question_options", "permissive"]
+
+    valid_examples = [
+        (
+            {"answer": {"Item1": 1, "Item2": 2}},
+            {
+                "question_items": ["Item1", "Item2"],
+                "question_options": [1, 2, 3],
+            },
+        )
+    ]
+
+    invalid_examples = [
+        (
+            {"answer": {"Item1": 1}},
+            {
+                "question_items": ["Item1", "Item2"],
+                "question_options": [1, 2, 3],
+            },
+            "Missing responses for some items",
+        ),
+        (
+            {"answer": {"Item1": 4, "Item2": 5}},
+            {
+                "question_items": ["Item1", "Item2"],
+                "question_options": [1, 2, 3],
+            },
+            "Invalid options selected",
+        ),
+    ]
+
+    def fix(self, response, verbose=False):
+        if verbose:
+            print(f"Fixing matrix response: {response}")
+
+        # If we have generated tokens, try to parse them
+        if "generated_tokens" in response:
+            try:
+                import json
+
+                fixed = json.loads(response["generated_tokens"])
+                if isinstance(fixed, dict):
+                    # Map numeric keys to question items
+                    mapped_answer = {}
+                    for idx, item in enumerate(self.question_items):
+                        if str(idx) in fixed:
+                            mapped_answer[item] = fixed[str(idx)]
+                    if (
+                        mapped_answer
+                    ):  # Only return if we successfully mapped some answers
+                        return {"answer": mapped_answer}
+            except:
+                pass
+
+        # If answer uses numeric keys, map them to question items
+        if "answer" in response and isinstance(response["answer"], dict):
+            if all(str(key).isdigit() for key in response["answer"].keys()):
+                mapped_answer = {}
+                for idx, item in enumerate(self.question_items):
+                    if str(idx) in response["answer"]:
+                        mapped_answer[item] = response["answer"][str(idx)]
+                if mapped_answer:  # Only update if we successfully mapped some answers
+                    response["answer"] = mapped_answer
+
+        return response
+
+
+class QuestionMatrix(QuestionBase):
+    """A question that presents a matrix/grid where multiple items are rated using the same scale."""
+
+    question_type = "matrix"
+    question_text: str = QuestionTextDescriptor()
+    question_items: List[str] = QuestionOptionsDescriptor()
+    question_options: List[Union[int, str, float]] = QuestionOptionsDescriptor()
+    option_labels: Optional[Dict[Union[int, str, float], str]] = OptionLabelDescriptor()
+
+    _response_model = None
+    response_validator_class = MatrixResponseValidator
+
+    def __init__(
+        self,
+        question_name: str,
+        question_text: str,
+        question_items: List[str],
+        question_options: List[Union[int, str, float]],
+        option_labels: Optional[Dict[Union[int, str, float], str]] = None,
+        include_comment: bool = True,
+        answering_instructions: Optional[str] = None,
+        question_presentation: Optional[str] = None,
+        permissive: bool = False,
+    ):
+        """Initialize a matrix question.
+
+        Args:
+            question_name: The name of the question
+            question_text: The text of the question
+            question_items: List of items to be rated
+            question_options: List of rating options
+            option_labels: Optional mapping of options to their labels
+            include_comment: Whether to include a comment field
+            answering_instructions: Optional custom instructions
+            question_presentation: Optional custom presentation
+            permissive: Whether to strictly validate responses
+        """
+        self.question_name = question_name
+
+        try:
+            self.question_text = question_text
+        except Exception as e:
+            raise QuestionCreationValidationError(
+                "question_text cannot be empty or too short!"
+            ) from e
+
+        self.question_items = question_items
+        self.question_options = question_options
+        self.option_labels = option_labels or {}
+
+        self.include_comment = include_comment
+        self.answering_instructions = answering_instructions
+        self.question_presentation = question_presentation
+        self.permissive = permissive
+
+    def create_response_model(self):
+        return create_matrix_response(
+            self.question_items, self.question_options, self.permissive
+        )
+
+    @property
+    def question_html_content(self) -> str:
+        """Generate HTML representation of the matrix question."""
+        template = Template(
+            """
+        <table class="matrix-question">
+            <tr>
+                <th></th>
+                {% for option in question_options %}
+                <th>
+                    {{ option }}
+                    {% if option in option_labels %}
+                    <br>
+                    <small>{{ option_labels[option] }}</small>
+                    {% endif %}
+                </th>
+                {% endfor %}
+            </tr>
+            {% for item in question_items %}
+            <tr>
+                <td>{{ item }}</td>
+                {% for option in question_options %}
+                <td>
+                    <input type="radio" 
+                           name="{{ question_name }}_{{ item }}" 
+                           value="{{ option }}"
+                           id="{{ question_name }}_{{ item }}_{{ option }}">
+                </td>
+                {% endfor %}
+            </tr>
+            {% endfor %}
+        </table>
+        """
+        )
+
+        return template.render(
+            question_name=self.question_name,
+            question_items=self.question_items,
+            question_options=self.question_options,
+            option_labels=self.option_labels,
+        )
+
+    @classmethod
+    @inject_exception
+    def example(cls) -> QuestionMatrix:
+        """Return an example matrix question."""
+        return cls(
+            question_name="child_happiness",
+            question_text="How happy would you be with different numbers of children?",
+            question_items=[
+                "No children",
+                "1 child",
+                "2 children",
+                "3 or more children",
+            ],
+            question_options=[1, 2, 3, 4, 5],
+            option_labels={1: "Very sad", 3: "Neutral", 5: "Extremely happy"},
+        )
+
+    def _simulate_answer(self) -> dict:
+        """Simulate a random valid answer."""
+        return {
+            "answer": {
+                item: random.choice(self.question_options)
+                for item in self.question_items
+            }
+        }
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/questions/QuestionMultipleChoice.py

@@ -120,9 +120,9 @@ class QuestionMultipleChoice(QuestionBase):
 
     question_type = "multiple_choice"
     purpose = "When options are known and limited"
-    question_options: Union[
-        list[str], list[list], list[float], list[int]
-    ] = QuestionOptionsDescriptor()
+    question_options: Union[list[str], list[list], list[float], list[int]] = (
+        QuestionOptionsDescriptor()
+    )
     _response_model = None
     response_validator_class = MultipleChoiceResponseValidator
 
@@ -175,8 +175,54 @@ class QuestionMultipleChoice(QuestionBase):
         else:
             return create_response_model(self.question_options, self.permissive)
 
+    @staticmethod
+    def _translate_question_options(
+        question_options, substitution_dict: dict
+    ) -> list[str]:
+
+        if isinstance(question_options, str):
+            # If dynamic options are provided like {{ options }}, render them with the scenario
+            # We can check if it's in the Scenario.
+            from jinja2 import Environment, meta
+
+            env = Environment()
+            parsed_content = env.parse(question_options)
+            template_variables = list(meta.find_undeclared_variables(parsed_content))
+            # print("The template variables are: ", template_variables)
+            question_option_key = template_variables[0]
+            # We need to deal with possibility it's actually an answer to a question.
+            potential_replacement = substitution_dict.get(question_option_key, None)
+
+            if isinstance(potential_replacement, list):
+                # translated_options = potential_replacement
+                return potential_replacement
+
+            if isinstance(potential_replacement, QuestionBase):
+                if hasattr(potential_replacement, "answer") and isinstance(
+                    potential_replacement.answer, list
+                ):
+                    return potential_replacement.answer
+                    # translated_options = potential_replacement.answer
+
+            # if not isinstance(potential_replacement, list):
+            # translated_options = potential_replacement
+
+            if potential_replacement is None:
+                # Nope - maybe it's in the substition dict?
+                raise ValueError(
+                    f"Could not find the key '{question_option_key}' in the scenario."
+                    f"The substition dict was: '{substitution_dict}.'"
+                    f"The question options were: '{question_options}'."
+                )
+        else:
+            translated_options = [
+                Template(str(option)).render(substitution_dict)
+                for option in question_options
+            ]
+        return translated_options
+
     def _translate_answer_code_to_answer(
-        self, answer_code: int, scenario: Optional["Scenario"] = None
+        self, answer_code: int, replacements_dict: Optional[dict] = None
     ):
         """Translate the answer code to the actual answer.
 
@@ -192,26 +238,24 @@ class QuestionMultipleChoice(QuestionBase):
         'Happy'
 
         """
+        if replacements_dict is None:
+            replacements_dict = {}
+        translated_options = self._translate_question_options(
+            self.question_options, replacements_dict
+        )
 
-        scenario = scenario or Scenario()
-
-        if isinstance(self.question_options, str):
-            # If dynamic options are provided like {{ options }}, render them with the scenario
-            from jinja2 import Environment, meta
-
-            env = Environment()
-            parsed_content = env.parse(self.question_options)
-            question_option_key = list(meta.find_undeclared_variables(parsed_content))[
-                0
-            ]
-            translated_options = scenario.get(question_option_key)
-        else:
-            translated_options = [
-                Template(str(option)).render(scenario)
-                for option in self.question_options
-            ]
         if self._use_code:
-            return translated_options[int(answer_code)]
+            try:
+                return translated_options[int(answer_code)]
+            except IndexError:
+                raise ValueError(
+                    f"Answer code is out of range. The answer code index was: {int(answer_code)}. The options were: {translated_options}."
+                )
+            except TypeError:
+                raise ValueError(
+                    f"The answer code was: '{answer_code}.'",
+                    f"The options were: '{translated_options}'.",
+                )
         else:
             # return translated_options[answer_code]
             return answer_code

edsl/questions/QuestionNumerical.py

@@ -1,17 +1,15 @@
 from __future__ import annotations
 
-# from decimal import Decimal
 from random import uniform
 from typing import Any, Optional, Union, Literal
 
 from pydantic import BaseModel, Field, field_validator
 
-from edsl.exceptions import QuestionAnswerValidationError
+from edsl.exceptions.questions import QuestionAnswerValidationError
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import NumericalOrNoneDescriptor
 from edsl.questions.decorators import inject_exception
 from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
-from edsl.exceptions.questions import QuestionAnswerValidationError
 
 
 def create_numeric_response(

edsl/questions/QuestionRank.py

@@ -1,25 +1,14 @@
 from __future__ import annotations
-import random
-import textwrap
-from jinja2 import Template
-from typing import Any, Optional, Union
-from edsl.questions.QuestionBase import QuestionBase
-from edsl.exceptions import QuestionAnswerValidationError
+from typing import Optional, Any, List, Annotated, Literal
+
+from pydantic import BaseModel, Field
 
+from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import (
     QuestionOptionsDescriptor,
     NumSelectionsDescriptor,
 )
-
-from edsl.prompts import Prompt
-
-from pydantic import field_validator
 from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
-from edsl.questions.ResponseValidatorABC import BaseResponse
-from edsl.exceptions import QuestionAnswerValidationError
-
-from pydantic import BaseModel, Field, create_model
-from typing import Optional, Any, List, Annotated, Literal
 
 
 def create_response_model(
@@ -201,7 +190,8 @@ class QuestionRank(QuestionBase):
         self, answer_codes, scenario: Scenario = None
     ) -> list[str]:
         """Translate the answer code to the actual answer."""
-        from edsl.scenarios import Scenario
+        from edsl.scenarios.Scenario import Scenario
+        from jinja2 import Template
 
         scenario = scenario or Scenario()
         translated_options = [

edsl/questions/ResponseValidatorABC.py

@@ -1,11 +1,11 @@
+import logging
 from abc import ABC, abstractmethod
-from pydantic import BaseModel, Field, field_validator
-
-# from decimal import Decimal
 from typing import Optional, Any, List, TypedDict
 
-from edsl.exceptions import QuestionAnswerValidationError
-from pydantic import ValidationError
+from pydantic import BaseModel, Field, field_validator, ValidationError
+
+from edsl.exceptions.questions import QuestionAnswerValidationError
+from edsl.questions.ExceptionExplainer import ExceptionExplainer
 
 
 class BaseResponse(BaseModel):
@@ -72,7 +72,8 @@ class ResponseValidatorABC(ABC):
         return self.override_answer if self.override_answer else data
 
     def _base_validate(self, data: RawEdslAnswerDict) -> BaseModel:
-        """This is the main validation function. It takes the response_model and checks the data against it, returning the instantiated model.
+        """This is the main validation function. It takes the response_model and checks the data against it,
+        returning the instantiated model.
 
         >>> rv = ResponseValidatorABC.example("numerical")
         >>> rv._base_validate({"answer": 42})
@@ -81,7 +82,9 @@ class ResponseValidatorABC(ABC):
         try:
             return self.response_model(**data)
         except ValidationError as e:
-            raise QuestionAnswerValidationError(e, data=data, model=self.response_model)
+            raise QuestionAnswerValidationError(
+                message=str(e), pydantic_error=e, data=data, model=self.response_model
+            )
 
     def post_validation_answer_convert(self, data):
         return data
@@ -128,10 +131,13 @@ class ResponseValidatorABC(ABC):
             edsl_answer_dict = self._extract_answer(pydantic_edsl_answer)
             return self._post_process(edsl_answer_dict)
         except QuestionAnswerValidationError as e:
-            if verbose:
-                print(f"Failed to validate {raw_edsl_answer_dict}; {str(e)}")
             return self._handle_exception(e, raw_edsl_answer_dict)
 
+    def human_explanation(self, e: QuestionAnswerValidationError):
+        explanation = ExceptionExplainer(e, model_response=e.data).explain()
+        return explanation
+        # return e
+
     def _handle_exception(self, e: Exception, raw_edsl_answer_dict) -> EdslAnswerDict:
         if self.fixes_tried == 0:
             self.original_exception = e
@@ -140,12 +146,18 @@ class ResponseValidatorABC(ABC):
             self.fixes_tried += 1
             fixed_data = self.fix(raw_edsl_answer_dict)
             try:
-                return self.validate(fixed_data, fix=True)
+                return self.validate(fixed_data, fix=True)  # early return if validates
             except Exception as e:
                 pass  # we don't log failed fixes
 
+        # If the exception is already a QuestionAnswerValidationError, raise it
+        if isinstance(self.original_exception, QuestionAnswerValidationError):
+            raise self.original_exception
+
+        # If nothing worked, raise the original exception
         raise QuestionAnswerValidationError(
-            self.original_exception,
+            message=self.original_exception,
+            pydantic_error=self.original_exception,
             data=raw_edsl_answer_dict,
             model=self.response_model,
         )
@@ -167,8 +179,22 @@ class ResponseValidatorABC(ABC):
         return q.response_validator
 
 
+def main():
+    rv = ResponseValidatorABC.example()
+    print(rv.validate({"answer": 42}))
+
+
 # Example usage
 if __name__ == "__main__":
     import doctest
 
     doctest.testmod(optionflags=doctest.ELLIPSIS)
+
+    rv = ResponseValidatorABC.example()
+    # print(rv.validate({"answer": 42}))
+
+    rv = ResponseValidatorABC.example()
+    try:
+        rv.validate({"answer": "120"})
+    except QuestionAnswerValidationError as e:
+        print(rv.human_explanation(e))

edsl/questions/ResponseValidatorFactory.py

@@ -0,0 +1,28 @@
+class ResponseValidatorFactory:
+    def __init__(self, question):
+        self.question = question
+
+    @property
+    def response_model(self) -> type["BaseModel"]:
+        if self.question._response_model is not None:
+            return self.question._response_model
+        else:
+            return self.question.create_response_model()
+
+    @property
+    def response_validator(self) -> "ResponseValidatorBase":
+        """Return the response validator."""
+        params = (
+            {
+                "response_model": self.question.response_model,
+            }
+            | {k: getattr(self.question, k) for k in self.validator_parameters}
+            | {"exception_to_throw": getattr(self.question, "exception_to_throw", None)}
+            | {"override_answer": getattr(self.question, "override_answer", None)}
+        )
+        return self.question.response_validator_class(**params)
+
+    @property
+    def validator_parameters(self) -> list[str]:
+        """Return the parameters required for the response validator."""
+        return self.question.response_validator_class.required_params

edsl/questions/SimpleAskMixin.py

@@ -1,5 +1,3 @@
-from rich.table import Table
-from rich.console import Console
 import math
 
 
@@ -10,6 +8,9 @@ def logprob_to_prob(logprob):
 
 
 def format_output(data):
+    from rich.table import Table
+    from rich.console import Console
+
     content = data["choices"][0]["logprobs"]["content"]
     table = Table(show_header=True, header_style="bold magenta")
 
@@ -65,7 +66,7 @@ class SimpleAskMixin:
         system_prompt="You are a helpful agent pretending to be a human. Do not break character",
         top_logprobs=4,
     ):
-        from edsl import Model
+        from edsl.language_models.registry import Model
 
         if model is None:
             model = Model()

edsl/questions/__init__.py

@@ -11,6 +11,7 @@ from edsl.questions.QuestionExtract import QuestionExtract
 from edsl.questions.QuestionFreeText import QuestionFreeText
 from edsl.questions.QuestionFunctional import QuestionFunctional
 from edsl.questions.QuestionList import QuestionList
+from edsl.questions.QuestionMatrix import QuestionMatrix
 from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
 from edsl.questions.QuestionNumerical import QuestionNumerical
 from edsl.questions.QuestionBudget import QuestionBudget

edsl/questions/derived/QuestionLinearScale.py

@@ -40,9 +40,12 @@ class QuestionLinearScale(QuestionMultipleChoice):
             include_comment=include_comment,
         )
         self.question_options = question_options
-        self.option_labels = (
-            {int(k): v for k, v in option_labels.items()} if option_labels else {}
-        )
+        if isinstance(option_labels, str):
+            self.option_labels = option_labels
+        else:
+            self.option_labels = (
+                {int(k): v for k, v in option_labels.items()} if option_labels else {}
+            )
         self.answering_instructions = answering_instructions
         self.question_presentation = question_presentation
 

edsl/questions/derived/QuestionTopK.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 from typing import Optional
 
-from edsl.exceptions import QuestionCreationValidationError
+from edsl.exceptions.questions import QuestionCreationValidationError
 from edsl.questions.QuestionCheckBox import QuestionCheckBox
 from edsl.questions.decorators import inject_exception
 

edsl/questions/descriptors.py

@@ -3,7 +3,7 @@
 from abc import ABC, abstractmethod
 import re
 from typing import Any, Callable, List, Optional
-from edsl.exceptions import (
+from edsl.exceptions.questions import (
     QuestionCreationValidationError,
     QuestionAnswerValidationError,
 )
@@ -181,11 +181,25 @@ class NumSelectionsDescriptor(BaseDescriptor):
 
 
 class OptionLabelDescriptor(BaseDescriptor):
-    """Validate that the `option_label` attribute is a string."""
+    """Validate that the `option_label` attribute is a string.
+
+    >>> class TestQuestion:
+    ...     option_label = OptionLabelDescriptor()
+    ...     def __init__(self, option_label: str):
+    ...         self.option_label = option_label
+
+    >>> _ = TestQuestion("{{Option}}")
+
+    """
 
     def validate(self, value, instance):
         """Validate the value is a string."""
-        # key_values = [int(v) for v in value.keys()]
+        if isinstance(value, str):
+            if "{{" in value and "}}" in value:
+                # they're trying to use a dynamic question name - let's let this play out
+                return None
+
+        key_values = [int(v) for v in value.keys()]
 
         if value and (key_values := [float(v) for v in value.keys()]) != []:
             if min(key_values) != min(instance.question_options):

edsl/questions/question_registry.py

@@ -96,7 +96,7 @@ class Question(metaclass=Meta):
 
         >>> from edsl import Question
         >>> Question.list_question_types()
-        ['checkbox', 'extract', 'free_text', 'functional', 'likert_five', 'linear_scale', 'list', 'multiple_choice', 'numerical', 'rank', 'top_k', 'yes_no']
+        ['checkbox', 'extract', 'free_text', 'functional', 'likert_five', 'linear_scale', 'list', 'matrix', 'multiple_choice', 'numerical', 'rank', 'top_k', 'yes_no']
         """
         return [
             q

edsl/questions/templates/matrix/__init__.py

@@ -0,0 +1 @@
+

edsl/questions/templates/matrix/answering_instructions.jinja

@@ -0,0 +1,5 @@
+Please respond with a dictionary mapping row codes to column codes. E.g., {"0": 1, "1": 3}
+
+{% if include_comment %}
+After the answer, you can put a comment explaining your choices on the next line.
+{% endif %}