edsl 0.1.33.dev2__py3-none-any.whl → 0.1.34__py3-none-any.whl

edsl/questions/QuestionBase.py

@@ -44,6 +44,13 @@ class QuestionBase(
     _answering_instructions = None
     _question_presentation = None
 
+    @property
+    def response_model(self) -> type["BaseModel"]:
+        if self._response_model is not None:
+            return self._response_model
+        else:
+            return self.create_response_model()
+
     # region: Validation and simulation methods
     @property
     def response_validator(self) -> "ResponseValidatorBase":
@@ -99,20 +106,17 @@ class QuestionBase(
         comment: Optional[str]
         generated_tokens: Optional[str]
 
-    def _validate_answer(self, answer: dict) -> ValidatedAnswer:
+    def _validate_answer(
+        self, answer: dict, replacement_dict: dict = None
+    ) -> ValidatedAnswer:
         """Validate the answer.
         >>> from edsl.exceptions import QuestionAnswerValidationError
         >>> from edsl import QuestionFreeText as Q
-        >>> Q.example()._validate_answer({'answer': 'Hello'})
-        {'answer': 'Hello', 'generated_tokens': None}
-        >>> Q.example()._validate_answer({'shmanswer': 1})
-        Traceback (most recent call last):
-        ...
-        edsl.exceptions.questions.QuestionAnswerValidationError:...
-        ...
+        >>> Q.example()._validate_answer({'answer': 'Hello', 'generated_tokens': 'Hello'})
+        {'answer': 'Hello', 'generated_tokens': 'Hello'}
         """
 
-        return self.response_validator.validate(answer)
+        return self.response_validator.validate(answer, replacement_dict)
 
     # endregion
 
@@ -471,6 +475,7 @@ class QuestionBase(
         self,
         scenario: Optional[dict] = None,
         agent: Optional[dict] = {},
+        answers: Optional[dict] = None,
         include_question_name: bool = False,
         height: Optional[int] = None,
         width: Optional[int] = None,
@@ -482,6 +487,17 @@ class QuestionBase(
         if scenario is None:
             scenario = {}
 
+        prior_answers_dict = {}
+
+        if isinstance(answers, dict):
+            for key, value in answers.items():
+                if not key.endswith("_comment") and not key.endswith(
+                    "_generated_tokens"
+                ):
+                    prior_answers_dict[key] = {"answer": value}
+
+        # breakpoint()
+
         base_template = """
         <div id="{{ question_name }}" class="survey_question" data-type="{{ question_type }}">
             {% if include_question_name %}
@@ -501,13 +517,40 @@ class QuestionBase(
 
         base_template = Template(base_template)
 
-        params = {
-            "question_name": self.question_name,
-            "question_text": Template(self.question_text).render(scenario, agent=agent),
-            "question_type": self.question_type,
-            "question_content": Template(question_content).render(scenario),
-            "include_question_name": include_question_name,
-        }
+        context = {
+            "scenario": scenario,
+            "agent": agent,
+        } | prior_answers_dict
+
+        # Render the question text
+        try:
+            question_text = Template(self.question_text).render(context)
+        except Exception as e:
+            print(
+                f"Error rendering question: question_text = {self.question_text}, error = {e}"
+            )
+            question_text = self.question_text
+
+        try:
+            question_content = Template(question_content).render(context)
+        except Exception as e:
+            print(
+                f"Error rendering question: question_content = {question_content}, error = {e}"
+            )
+            question_content = question_content
+
+        try:
+            params = {
+                "question_name": self.question_name,
+                "question_text": question_text,
+                "question_type": self.question_type,
+                "question_content": question_content,
+                "include_question_name": include_question_name,
+            }
+        except Exception as e:
+            raise ValueError(
+                f"Error rendering question: params = {params}, error = {e}"
+            )
         rendered_html = base_template.render(**params)
 
         if iframe:
@@ -526,6 +569,21 @@ class QuestionBase(
 
         return rendered_html
 
+    @classmethod
+    def example_model(cls):
+        from edsl import Model
+
+        q = cls.example()
+        m = Model("test", canned_response=cls._simulate_answer(q)["answer"])
+
+        return m
+
+    @classmethod
+    def example_results(cls):
+        m = cls.example_model()
+        q = cls.example()
+        return q.by(m).run(cache=False)
+
     def rich_print(self):
         """Print the question in a rich format."""
         from rich.table import Table

edsl/questions/QuestionBaseGenMixin.py

@@ -95,6 +95,34 @@ class QuestionBaseGenMixin:
             questions.append(QuestionBase.from_dict(new_data))
         return questions
 
+    def render(self, replacement_dict: dict) -> "QuestionBase":
+        """Render the question components as jinja2 templates with the replacement dictionary."""
+        from jinja2 import Environment
+        from edsl import Scenario
+
+        strings_only_replacement_dict = {
+            k: v for k, v in replacement_dict.items() if not isinstance(v, Scenario)
+        }
+
+        def render_string(value: str) -> str:
+            if value is None or not isinstance(value, str):
+                return value
+            else:
+                try:
+                    return (
+                        Environment()
+                        .from_string(value)
+                        .render(strings_only_replacement_dict)
+                    )
+                except Exception as e:
+                    import warnings
+
+                    warnings.warn("Failed to render string: " + value)
+                    # breakpoint()
+                    return value
+
+        return self.apply_function(render_string)
+
     def apply_function(self, func: Callable, exclude_components=None) -> QuestionBase:
         """Apply a function to the question parts
 

edsl/questions/QuestionBudget.py

@@ -1,8 +1,60 @@
 from __future__ import annotations
-import random
-from typing import Any, Optional, Union
+from typing import Any, Optional, Union, List
+
+from pydantic import Field, BaseModel, validator
+
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import IntegerDescriptor, QuestionOptionsDescriptor
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+
+
+class BudgewResponseValidator(ResponseValidatorABC):
+    valid_examples = []
+
+    invalid_examples = []
+
+    def fix(self, response, verbose=False):
+        if verbose:
+            print(f"Fixing list response: {response}")
+        answer = str(response.get("answer") or response.get("generated_tokens", ""))
+        if len(answer.split(",")) > 0:
+            return (
+                {"answer": answer.split(",")} | {"comment": response.get("comment")}
+                if "comment" in response
+                else {}
+            )
+
+
+def create_budget_model(
+    budget_sum: float, permissive: bool, question_options: List[str]
+):
+    class BudgetResponse(BaseModel):
+        answer: List[float] = Field(
+            ...,
+            description="List of non-negative numbers representing budget allocation",
+            min_items=len(question_options),
+            max_items=len(question_options),
+        )
+        comment: Optional[str] = None
+        generated_tokens: Optional[str] = None
+
+        @validator("answer")
+        def validate_answer(cls, v):
+            if len(v) != len(question_options):
+                raise ValueError(f"Must provide {len(question_options)} values")
+            if any(x < 0 for x in v):
+                raise ValueError("All values must be non-negative")
+            total = sum(v)
+            if not permissive and total != budget_sum:
+                raise ValueError(f"Sum of numbers must equal {budget_sum}")
+            elif permissive and total > budget_sum:
+                raise ValueError(f"Sum of numbers cannot exceed {budget_sum}")
+            return v
+
+        class Config:
+            extra = "forbid"
+
+    return BudgetResponse
 
 
 class QuestionBudget(QuestionBase):
@@ -12,7 +64,7 @@ class QuestionBudget(QuestionBase):
     budget_sum: int = IntegerDescriptor(none_allowed=False)
     question_options: list[str] = QuestionOptionsDescriptor(q_budget=True)
     _response_model = None
-    response_validator_class = None
+    response_validator_class = BudgewResponseValidator
 
     def __init__(
         self,
@@ -20,8 +72,10 @@ class QuestionBudget(QuestionBase):
         question_text: str,
         question_options: list[str],
         budget_sum: int,
+        include_comment: bool = True,
         question_presentation: Optional[str] = None,
         answering_instructions: Optional[str] = None,
+        permissive: bool = False,
     ):
         """Instantiate a new QuestionBudget.
 
@@ -36,20 +90,17 @@ class QuestionBudget(QuestionBase):
         self.budget_sum = budget_sum
         self.question_presentation = question_presentation
         self.answering_instructions = answering_instructions
+        self.permissive = permissive
+        self.include_comment = include_comment
 
-    ################
-    # Answer methods
-    ################
-    def _validate_answer(self, answer: dict[str, Any]) -> dict[str, Union[int, str]]:
-        """Validate the answer."""
-        self._validate_answer_template_basic(answer)
-        self._validate_answer_key_value(answer, "answer", dict)
-        self._validate_answer_budget(answer)
-        return answer
+    def create_response_model(self):
+        return create_budget_model(
+            self.budget_sum, self.permissive, self.question_options
+        )
 
     def _translate_answer_code_to_answer(
-        self, answer_codes: dict[str, int], scenario: "Scenario" = None
-    ):
+        self, answer_code, combined_dict
+    ) -> list[dict]:
         """
         Translate the answer codes to the actual answers.
 
@@ -58,35 +109,35 @@ class QuestionBudget(QuestionBase):
         This code will translate that to "a".
         """
         translated_codes = []
-        for answer_code, response in answer_codes.items():
-            translated_codes.append({self.question_options[int(answer_code)]: response})
+        for answer_code, question_option in zip(answer_code, self.question_options):
+            translated_codes.append({question_option: answer_code})
 
         return translated_codes
 
-    def _simulate_answer(self, human_readable=True):
-        """Simulate a valid answer for debugging purposes (what the validator expects)."""
-        from edsl.utilities.utilities import random_string
-
-        if human_readable:
-            keys = self.question_options
-        else:
-            keys = range(len(self.question_options))
-        remaining_budget = self.budget_sum
-        values = []
-        for _ in range(len(self.question_options)):
-            if _ == len(self.question_options) - 1:
-                # Assign remaining budget to the last value
-                values.append(remaining_budget)
-            else:
-                # Generate a random value between 0 and remaining budget
-                value = random.randint(0, remaining_budget)
-                values.append(value)
-                remaining_budget -= value
-        answer = dict(zip(keys, values))
-        return {
-            "answer": answer,
-            "comment": random_string(),
-        }
+    # def _simulate_answer(self, human_readable=True):
+    #     """Simulate a valid answer for debugging purposes (what the validator expects)."""
+    #     from edsl.utilities.utilities import random_string
+
+    #     if human_readable:
+    #         keys = self.question_options
+    #     else:
+    #         keys = range(len(self.question_options))
+    #     remaining_budget = self.budget_sum
+    #     values = []
+    #     for _ in range(len(self.question_options)):
+    #         if _ == len(self.question_options) - 1:
+    #             # Assign remaining budget to the last value
+    #             values.append(remaining_budget)
+    #         else:
+    #             # Generate a random value between 0 and remaining budget
+    #             value = random.randint(0, remaining_budget)
+    #             values.append(value)
+    #             remaining_budget -= value
+    #     answer = dict(zip(keys, values))
+    #     return {
+    #         "answer": answer,
+    #         "comment": random_string(),
+    #     }
 
     @property
     def question_html_content(self) -> str:
@@ -133,13 +184,14 @@ class QuestionBudget(QuestionBase):
     # Helpful methods
     ################
     @classmethod
-    def example(cls) -> QuestionBudget:
+    def example(cls, include_comment: bool = True) -> QuestionBudget:
         """Return an example of a budget question."""
         return cls(
             question_name="food_budget",
             question_text="How would you allocate $100?",
             question_options=["Pizza", "Ice Cream", "Burgers", "Salad"],
             budget_sum=100,
+            include_comment=include_comment,
         )
 
 

edsl/questions/QuestionCheckBox.py

@@ -245,7 +245,7 @@ class QuestionCheckBox(QuestionBase):
 
         scenario = scenario or Scenario()
         translated_options = [
-            Template(option).render(scenario) for option in self.question_options
+            Template(str(option)).render(scenario) for option in self.question_options
         ]
         translated_codes = []
         for answer_code in answer_codes:

edsl/questions/QuestionFreeText.py

@@ -36,6 +36,12 @@ class FreeTextResponseValidator(ResponseValidatorABC):
         ),
     ]
 
+    def fix(self, response, verbose=False):
+        return {
+            "answer": str(response.get("generated_tokens")),
+            "generated_tokens": str(response.get("generated_tokens")),
+        }
+
 
 class QuestionFreeText(QuestionBase):
     """This question prompts the agent to respond with free text."""

edsl/questions/QuestionMultipleChoice.py

@@ -1,22 +1,14 @@
 from __future__ import annotations
-from typing import Union, Literal, Optional
-from jinja2 import Template
+from typing import Union, Literal, Optional, List, Any
 
+from jinja2 import Template
 from pydantic import BaseModel, Field
-from typing import Optional, Literal
 
+from edsl.scenarios.Scenario import Scenario
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import QuestionOptionsDescriptor
 from edsl.questions.decorators import inject_exception
-
 from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
-from edsl.questions.ResponseValidatorABC import BaseResponse
-
-from edsl.exceptions import QuestionAnswerValidationError
-
-from pydantic import BaseModel, Field, create_model
-
-from typing import List, Any, Literal
 
 
 def create_response_model(choices: List[str], permissive: bool = False):
@@ -27,7 +19,6 @@ def create_response_model(choices: List[str], permissive: bool = False):
     :param permissive: If True, any value will be accepted as an answer.
     :return: A new Pydantic model class.
     """
-    # Convert the choices list to a tuple for use with Literal
     choice_tuple = tuple(choices)
 
     if not permissive:
@@ -66,16 +57,6 @@ def create_response_model(choices: List[str], permissive: bool = False):
     return ChoiceResponse
 
 
-def fix_multiple_choice(response, question_options, use_code, verbose=False):
-    """Fix the response to a multiple choice question.
-    Respnse is a dictionary with keys:
-    - answer: the answer code
-    - generated_tokens: the generated tokens
-    - comment: the comment
-    """
-    pass
-
-
 class MultipleChoiceResponseValidator(ResponseValidatorABC):
     required_params = ["question_options", "use_code"]
 
@@ -161,6 +142,11 @@ class QuestionMultipleChoice(QuestionBase):
         :param question_name: The name of the question.
         :param question_text: The text of the question.
         :param question_options: The options the agent should select from.
+        :param include_comment: Whether to include a comment field.
+        :param use_code: Whether to use code for the options.
+        :param answering_instructions: Instructions for the question.
+        :param question_presentation: The presentation of the question.
+        :param permissive: Whether to force the answer to be one of the options.
 
         """
         self.question_name = question_name
@@ -177,7 +163,11 @@ class QuestionMultipleChoice(QuestionBase):
     # Answer methods
     ################
 
-    def create_response_model(self):
+    def create_response_model(self, replacement_dict: dict = None):
+        if replacement_dict is None:
+            replacement_dict = {}
+            # The replacement dict that could be from scenario, current answers, etc. to populate the response model
+
         if self.use_code:
             return create_response_model(
                 list(range(len(self.question_options))), self.permissive
@@ -202,7 +192,6 @@ class QuestionMultipleChoice(QuestionBase):
         'Happy'
 
         """
-        from edsl.scenarios.Scenario import Scenario
 
         scenario = scenario or Scenario()
 

edsl/questions/QuestionNumerical.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
-from decimal import Decimal
+
+# from decimal import Decimal
 from random import uniform
 from typing import Any, Optional, Union, Literal
 
@@ -14,8 +15,8 @@ from edsl.exceptions.questions import QuestionAnswerValidationError
 
 
 def create_numeric_response(
-    min_value: Optional[Decimal] = None,
-    max_value: Optional[Decimal] = None,
+    min_value: Optional[float] = None,
+    max_value: Optional[float] = None,
     permissive=False,
 ):
     field_kwargs = {}
@@ -27,7 +28,7 @@ def create_numeric_response(
             field_kwargs["le"] = max_value
 
     class ConstrainedNumericResponse(BaseModel):
-        answer: Union[Decimal] = Field(**field_kwargs)
+        answer: Union[int, float] = Field(**field_kwargs)
         comment: Optional[str] = Field(None)
         generated_tokens: Optional[Any] = Field(None)
 

edsl/questions/Quick.py

@@ -0,0 +1,41 @@
+from edsl import (
+    QuestionFreeText,
+    QuestionMultipleChoice,
+    Survey,
+    QuestionList,
+    Question,
+)
+
+
+def Quick(question_text):
+    q_type = QuestionMultipleChoice(
+        question_text=f"A researcher is asking a language model this: {question_text}. What is the most appropriate type of question to ask?",
+        question_name="potential_question_type",
+        question_options=["multiple_choice", "list", "free_text"],
+    )
+
+    q_name = QuestionFreeText(
+        question_text=f"A researcher is asking a language model this: {question_text}. What is a good name for this question that's a valid python identifier? Just return the proposed identifer",
+        question_name="potential_question_name",
+    )
+
+    q_options = QuestionList(
+        question_text=f"A research is asking this question: { question_text }. What are the possible options for this question?",
+        question_name="potential_question_options",
+    )
+
+    survey = Survey([q_type, q_name, q_options]).add_skip_rule(
+        q_options, "{{ potential_question_type }} != 'multiple_choice'"
+    )
+    return survey
+    # results = survey.run()
+    # question_type = results.select("potential_question_type").first()
+    # question_options = results.select("potential_question_options").first()
+    # question_name = results.select("potential_question_name").first()
+    # print("Question Type: ", question_type)
+    # print("Question Name: ", question_name)
+    # print("Question Options: ", question_options)
+    # if question_options == None:
+    #     return Question(question_type, question_name = question_name)
+    # else:
+    #     return Question(question_type, question_name = question_name, question_options = question_options)

edsl/questions/ResponseValidatorABC.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 from pydantic import BaseModel, Field, field_validator
-from decimal import Decimal
+
+# from decimal import Decimal
 from typing import Optional, Any, List, TypedDict
 
 from edsl.exceptions import QuestionAnswerValidationError
@@ -64,7 +65,7 @@ class ResponseValidatorABC(ABC):
         >>> rv = ResponseValidatorABC.example()
         >>> rv.override_answer = {"answer": 42}
         >>> rv.validate({"answer": 23})
-        {'answer': Decimal('42'), 'comment': None, 'generated_tokens': None}
+        {'answer': 42, 'comment': None, 'generated_tokens': None}
         """
         if self.exception_to_throw:
             raise self.exception_to_throw
@@ -75,7 +76,7 @@ class ResponseValidatorABC(ABC):
 
         >>> rv = ResponseValidatorABC.example("numerical")
         >>> rv._base_validate({"answer": 42})
-        ConstrainedNumericResponse(answer=Decimal('42'), comment=None, generated_tokens=None)
+        ConstrainedNumericResponse(answer=42, comment=None, generated_tokens=None)
         """
         try:
             return self.response_model(**data)
@@ -91,13 +92,17 @@ class ResponseValidatorABC(ABC):
         generated_tokens: Optional[str]
 
     def validate(
-        self, raw_edsl_answer_dict: RawEdslAnswerDict, fix=False, verbose=False
+        self,
+        raw_edsl_answer_dict: RawEdslAnswerDict,
+        fix=False,
+        verbose=False,
+        replacement_dict: dict = None,
     ) -> EdslAnswerDict:
         """This is the main validation function.
 
         >>> rv = ResponseValidatorABC.example("numerical")
         >>> rv.validate({"answer": 42})
-        {'answer': Decimal('42'), 'comment': None, 'generated_tokens': None}
+        {'answer': 42, 'comment': None, 'generated_tokens': None}
         >>> rv.max_value
         86.7
         >>> rv.validate({"answer": "120"})
@@ -109,7 +114,7 @@ class ResponseValidatorABC(ABC):
         >>> q.permissive = True
         >>> rv = q.response_validator
         >>> rv.validate({"answer": "120"})
-        {'answer': Decimal('120'), 'comment': None, 'generated_tokens': None}
+        {'answer': 120, 'comment': None, 'generated_tokens': None}
         >>> rv.validate({"answer": "poo"})
         Traceback (most recent call last):
         ...

edsl/questions/derived/QuestionLinearScale.py

@@ -22,6 +22,7 @@ class QuestionLinearScale(QuestionMultipleChoice):
         option_labels: Optional[dict[int, str]] = None,
         answering_instructions: Optional[str] = None,
         question_presentation: Optional[str] = None,
+        include_comment: Optional[bool] = True,
     ):
         """Instantiate a new QuestionLinearScale.
 
@@ -36,6 +37,7 @@ class QuestionLinearScale(QuestionMultipleChoice):
             question_text=question_text,
             question_options=question_options,
             use_code=False,  # question linear scale will have it's own code
+            include_comment=include_comment,
         )
         self.question_options = question_options
         self.option_labels = (
@@ -49,13 +51,14 @@ class QuestionLinearScale(QuestionMultipleChoice):
     ################
     @classmethod
     @inject_exception
-    def example(cls) -> QuestionLinearScale:
+    def example(cls, include_comment: bool = True) -> QuestionLinearScale:
         """Return an example of a linear scale question."""
         return cls(
             question_text="How much do you like ice cream?",
             question_options=[1, 2, 3, 4, 5],
             question_name="ice_cream",
             option_labels={1: "I hate it", 5: "I love it"},
+            include_comment=include_comment,
         )
 
 

edsl/questions/derived/QuestionTopK.py

@@ -20,6 +20,7 @@ class QuestionTopK(QuestionCheckBox):
         max_selections: int,
         question_presentation: Optional[str] = None,
         answering_instructions: Optional[str] = None,
+        include_comment: Optional[bool] = True,
     ):
         """Initialize the question.
 
@@ -37,6 +38,7 @@ class QuestionTopK(QuestionCheckBox):
             max_selections=max_selections,
             question_presentation=question_presentation,
             answering_instructions=answering_instructions,
+            include_comment=include_comment,
         )
         if min_selections != max_selections:
             raise QuestionCreationValidationError(
@@ -52,7 +54,7 @@ class QuestionTopK(QuestionCheckBox):
     ################
     @classmethod
     @inject_exception
-    def example(cls) -> QuestionTopK:
+    def example(cls, include_comment: bool = True) -> QuestionTopK:
         """Return an example question."""
         return cls(
             question_name="two_fruits",
@@ -60,6 +62,7 @@ class QuestionTopK(QuestionCheckBox):
             question_options=["apple", "banana", "carrot", "durian"],
             min_selections=2,
             max_selections=2,
+            include_comment=include_comment,
         )