edsl 0.1.14__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/questions/QuestionBasePromptsMixin.py

@@ -0,0 +1,221 @@
+from importlib import resources
+from typing import Optional
+from edsl.exceptions.questions import QuestionAnswerValidationError
+from functools import lru_cache
+
+
+class TemplateManager:
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._template_cache = {}
+        return cls._instance
+
+    @lru_cache(maxsize=None)
+    def get_template(self, question_type, template_name):
+        if (question_type, template_name) not in self._template_cache:
+            with resources.open_text(
+                f"edsl.questions.templates.{question_type}", template_name
+            ) as file:
+                self._template_cache[(question_type, template_name)] = file.read()
+        return self._template_cache[(question_type, template_name)]
+
+
+# Global instance
+template_manager = TemplateManager()
+
+
+class QuestionBasePromptsMixin:
+    @property
+    def model_instructions(self) -> dict:
+        """Get the model-specific instructions for the question."""
+        if not hasattr(self, "_model_instructions"):
+            self._model_instructions = {}
+        return self._model_instructions
+
+    def _all_text(self) -> str:
+        """Return the question text.
+
+        >>> from edsl import QuestionMultipleChoice as Q
+        >>> Q.example()._all_text()
+        "how_feelingHow are you?['Good', 'Great', 'OK', 'Bad']"
+        """
+        txt = ""
+        for key, value in self.data.items():
+            if isinstance(value, str):
+                txt += value
+            elif isinstance(value, list):
+                txt += "".join(str(value))
+        return txt
+
+    @model_instructions.setter
+    def model_instructions(self, data: dict):
+        """Set the model-specific instructions for the question."""
+        self._model_instructions = data
+
+    def add_model_instructions(
+        self, *, instructions: str, model: Optional[str] = None
+    ) -> None:
+        """Add model-specific instructions for the question that override the default instructions.
+
+        :param instructions: The instructions to add. This is typically a jinja2 template.
+        :param model: The language model for this instruction.
+
+        >>> from edsl.questions import QuestionFreeText
+        >>> q = QuestionFreeText(question_name = "color", question_text = "What is your favorite color?")
+        >>> q.add_model_instructions(instructions = "{{question_text}}. Answer in valid JSON like so {'answer': 'comment: <>}", model = "gpt3")
+        >>> q.get_instructions(model = "gpt3")
+        Prompt(text=\"""{{question_text}}. Answer in valid JSON like so {'answer': 'comment: <>}\""")
+        """
+        from edsl.language_models.model import Model
+
+        if not hasattr(self, "_model_instructions"):
+            self._model_instructions = {}
+        if model is None:
+            # if not model is passed, all the models are mapped to this instruction, including 'None'
+            self._model_instructions = {
+                model_name: instructions
+                for model_name in Model.available(name_only=True)
+            }
+            self._model_instructions.update({model: instructions})
+        else:
+            self._model_instructions.update({model: instructions})
+
+    @classmethod
+    def path_to_folder(cls) -> str:
+        return resources.files(f"edsl.questions.templates", cls.question_type)
+
+    @property
+    def response_model(self) -> type["BaseModel"]:
+        if self._response_model is not None:
+            return self._response_model
+        else:
+            return self.create_response_model()
+
+    @property
+    def use_code(self) -> bool:
+        if hasattr(self, "_use_code"):
+            return self._use_code
+        return True
+
+    @use_code.setter
+    def use_code(self, value: bool) -> None:
+        self._use_code = value
+
+    @property
+    def include_comment(self) -> bool:
+        if hasattr(self, "_include_comment"):
+            return self._include_comment
+        return True
+
+    @include_comment.setter
+    def include_comment(self, value: bool) -> None:
+        self._include_comment = value
+
+    @classmethod
+    def default_answering_instructions(cls) -> str:
+        # template_text = cls._read_template("answering_instructions.jinja")
+        template_text = template_manager.get_template(
+            cls.question_type, "answering_instructions.jinja"
+        )
+        from edsl.prompts import Prompt
+
+        return Prompt(text=template_text)
+
+    @classmethod
+    def default_question_presentation(cls):
+        template_text = template_manager.get_template(
+            cls.question_type, "question_presentation.jinja"
+        )
+        from edsl.prompts import Prompt
+
+        return Prompt(text=template_text)
+
+    @property
+    def answering_instructions(self) -> str:
+        if self._answering_instructions is None:
+            return self.default_answering_instructions()
+        return self._answering_instructions
+
+    @answering_instructions.setter
+    def answering_instructions(self, value) -> None:
+        self._answering_instructions = value
+
+    @property
+    def question_presentation(self):
+        if self._question_presentation is None:
+            return self.default_question_presentation()
+        return self._question_presentation
+
+    @question_presentation.setter
+    def question_presentation(self, value):
+        self._question_presentation = value
+
+    def prompt_preview(self, scenario=None, agent=None):
+        return self.new_default_instructions.render(
+            self.data
+            | {
+                "include_comment": getattr(self, "_include_comment", True),
+                "use_code": getattr(self, "_use_code", True),
+            }
+            | ({"scenario": scenario} or {})
+            | ({"agent": agent} or {})
+        )
+
+    @classmethod
+    def self_check(cls):
+        q = cls.example()
+        for answer, params in q.response_validator.valid_examples:
+            for key, value in params.items():
+                setattr(q, key, value)
+            q._validate_answer(answer)
+        for answer, params, reason in q.response_validator.invalid_examples:
+            for key, value in params.items():
+                setattr(q, key, value)
+            try:
+                q._validate_answer(answer)
+            except QuestionAnswerValidationError:
+                pass
+            else:
+                raise ValueError(f"Example {answer} should have failed for {reason}.")
+
+    @property
+    def new_default_instructions(self) -> "Prompt":
+        "This is set up as a property because there are mutable question values that determine how it is rendered."
+        from edsl.prompts import Prompt
+
+        return Prompt(self.question_presentation) + Prompt(self.answering_instructions)
+
+    @property
+    def parameters(self) -> set[str]:
+        """Return the parameters of the question."""
+        from jinja2 import Environment, meta
+
+        env = Environment()
+        # Parse the template
+        txt = self._all_text()
+        # txt = self.question_text
+        # if hasattr(self, "question_options"):
+        #    txt += " ".join(self.question_options)
+        parsed_content = env.parse(txt)
+        # Extract undeclared variables
+        variables = meta.find_undeclared_variables(parsed_content)
+        # Return as a list
+        return set(variables)
+
+    def get_instructions(self, model: Optional[str] = None) -> type["PromptBase"]:
+        """Get the mathcing question-answering instructions for the question.
+
+        :param model: The language model to use.
+        """
+        from edsl.prompts.Prompt import Prompt
+
+        if model in self.model_instructions:
+            return Prompt(text=self.model_instructions[model])
+        else:
+            if hasattr(self, "new_default_instructions"):
+                return self.new_default_instructions
+            else:
+                return self.applicable_prompts(model)[0]()

edsl/questions/QuestionBudget.py

@@ -1,33 +1,70 @@
 from __future__ import annotations
-import random
-import textwrap
-from typing import Any, Optional, Union
-from edsl.questions import Question
+from typing import Any, Optional, Union, List
+
+from pydantic import Field, BaseModel, validator
+
+from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import IntegerDescriptor, QuestionOptionsDescriptor
-from edsl.scenarios import Scenario
-from edsl.utilities import random_string
+from edsl.questions.response_validator_abc import ResponseValidatorABC
+
+
+class BudgetResponseValidator(ResponseValidatorABC):
+    valid_examples = []
+
+    invalid_examples = []
+
+    def fix(self, response, verbose=False):
+        if verbose:
+            print(f"Fixing list response: {response}")
+        answer = str(response.get("answer") or response.get("generated_tokens", ""))
+        if len(answer.split(",")) > 0:
+            return (
+                {"answer": answer.split(",")} | {"comment": response.get("comment")}
+                if "comment" in response
+                else {}
+            )
+
+
+def create_budget_model(
+    budget_sum: float, permissive: bool, question_options: List[str]
+):
+    class BudgetResponse(BaseModel):
+        answer: List[float] = Field(
+            ...,
+            description="List of non-negative numbers representing budget allocation",
+            min_items=len(question_options),
+            max_items=len(question_options),
+        )
+        comment: Optional[str] = None
+        generated_tokens: Optional[str] = None
 
+        @validator("answer")
+        def validate_answer(cls, v):
+            if len(v) != len(question_options):
+                raise ValueError(f"Must provide {len(question_options)} values")
+            if any(x < 0 for x in v):
+                raise ValueError("All values must be non-negative")
+            total = sum(v)
+            if not permissive and total != budget_sum:
+                raise ValueError(f"Sum of numbers must equal {budget_sum}")
+            elif permissive and total > budget_sum:
+                raise ValueError(f"Sum of numbers cannot exceed {budget_sum}")
+            return v
 
-class QuestionBudget(Question):
-    """
-    This question asks the respondent to allocate a budget among options.
+        class Config:
+            extra = "forbid"
 
-    Arguments:
-    - `budget_sum` is the total amount of the budget to be allocated (positive integer)
-    - `question_name` is the name of the question (string)
-    - `question_options` are the options the user should allocated the budget to (list of strings)
-    - `question_text` is the text of the question (string)
+    return BudgetResponse
 
-    Optional arguments:
-    - `instructions` are the instructions for the question (string). If not provided, the default instructions are used. To view them, run `QuestionBudget.default_instructions`
-    - `short_names_dict` maps question_options to short names (dictionary mapping strings to strings)
 
-    To generate an example, run `QuestionBudget.example()`
-    """
+class QuestionBudget(QuestionBase):
+    """This question prompts the agent to allocate a budget among options."""
 
     question_type = "budget"
     budget_sum: int = IntegerDescriptor(none_allowed=False)
-    question_options: list[str] = QuestionOptionsDescriptor()
+    question_options: list[str] = QuestionOptionsDescriptor(q_budget=True)
+    _response_model = None
+    response_validator_class = BudgetResponseValidator
 
     def __init__(
         self,
@@ -35,96 +72,156 @@ class QuestionBudget(Question):
         question_text: str,
         question_options: list[str],
         budget_sum: int,
-        short_names_dict: Optional[dict[str, str]] = None,
+        include_comment: bool = True,
+        question_presentation: Optional[str] = None,
+        answering_instructions: Optional[str] = None,
+        permissive: bool = False,
     ):
+        """Instantiate a new QuestionBudget.
+
+        :param question_name: The name of the question.
+        :param question_text: The text of the question.
+        :param question_options: The options for allocation of the budget sum.
+        :param budget_sum: The total amount of the budget to be allocated among the options.
+        """
         self.question_name = question_name
         self.question_text = question_text
         self.question_options = question_options
         self.budget_sum = budget_sum
-        self.short_names_dict = short_names_dict or {}
+        self.question_presentation = question_presentation
+        self.answering_instructions = answering_instructions
+        self.permissive = permissive
+        self.include_comment = include_comment
+
+    def create_response_model(self):
+        return create_budget_model(
+            self.budget_sum, self.permissive, self.question_options
+        )
+
+    def _translate_answer_code_to_answer(
+        self, answer_code, combined_dict
+    ) -> list[dict]:
+        """
+        Translate the answer codes to the actual answers.
 
-    ################
-    # Answer methods
-    ################
-    def validate_answer(self, answer: dict[str, Any]) -> dict[str, Union[int, str]]:
-        self.validate_answer_template_basic(answer)
-        self.validate_answer_key_value(answer, "answer", dict)
-        self.validate_answer_budget(answer)
-        return answer
-
-    def translate_answer_code_to_answer(
-        self, answer_codes: dict[str, int], scenario: Scenario = None
-    ):
-        """Translates the answer codes to the actual answers.
         For example, for a budget question with options ["a", "b", "c"],
         the answer codes are 0, 1, and 2. The LLM will respond with 0.
         This code will translate that to "a".
         """
         translated_codes = []
-        for answer_code, response in answer_codes.items():
-            translated_codes.append({self.question_options[int(answer_code)]: response})
+        for answer_code, question_option in zip(answer_code, self.question_options):
+            translated_codes.append({question_option: answer_code})
 
         return translated_codes
 
-    def simulate_answer(self, human_readable=True):
-        "Simulates a valid answer for debugging purposes (what the validator expects)"
-        if human_readable:
-            keys = self.question_options
-        else:
-            keys = range(len(self.question_options))
-        remaining_budget = self.budget_sum
-        values = []
-        for _ in range(len(self.question_options)):
-            if _ == len(self.question_options) - 1:
-                # Assign remaining budget to the last value
-                values.append(remaining_budget)
-            else:
-                # Generate a random value between 0 and remaining budget
-                value = random.randint(0, remaining_budget)
-                values.append(value)
-                remaining_budget -= value
-        answer = dict(zip(keys, values))
-        return {
-            "answer": answer,
-            "comment": random_string(),
+    # def _simulate_answer(self, human_readable=True):
+    #     """Simulate a valid answer for debugging purposes (what the validator expects)."""
+    #     from edsl.utilities.utilities import random_string
+
+    #     if human_readable:
+    #         keys = self.question_options
+    #     else:
+    #         keys = range(len(self.question_options))
+    #     remaining_budget = self.budget_sum
+    #     values = []
+    #     for _ in range(len(self.question_options)):
+    #         if _ == len(self.question_options) - 1:
+    #             # Assign remaining budget to the last value
+    #             values.append(remaining_budget)
+    #         else:
+    #             # Generate a random value between 0 and remaining budget
+    #             value = random.randint(0, remaining_budget)
+    #             values.append(value)
+    #             remaining_budget -= value
+    #     answer = dict(zip(keys, values))
+    #     return {
+    #         "answer": answer,
+    #         "comment": random_string(),
+    #     }
+
+    @property
+    def question_html_content(self) -> str:
+        from jinja2 import Template
+
+        question_html_content = Template(
+            """
+        <form id="budgetForm">
+        <p>Total Budget: {{ budget_sum }}</p>
+        <p>Remaining Budget: <span id="remainingBudget">{{ budget_sum }}</span></p>
+        {% for option in question_options %}
+        <div>
+            <label for="{{ option }}">{{ option }}</label>
+            <input type="number" id="{{ option }}" name="{{ question_name }}[{{ option }}]" value="0" min="0" max="{{ budget_sum }}" oninput="updateRemainingBudget()">
+        </div>
+        {% endfor %}
+        </form>
+        <script>
+        function updateRemainingBudget() {
+            let totalBudget = {{ budget_sum }};
+            let allocated = 0;
+
+            {% for option in question_options %}
+            allocated += parseInt(document.getElementById("{{ option }}").value) || 0;
+            {% endfor %}
+
+            let remaining = totalBudget - allocated;
+            document.getElementById('remainingBudget').innerText = remaining;
+
+            {% for option in question_options %}
+            document.getElementById("{{ option }}").max = remaining + parseInt(document.getElementById("{{ option }}").value);
+            {% endfor %}
         }
+        </script>
+        """
+        ).render(
+            question_name=self.question_name,
+            budget_sum=self.budget_sum,
+            question_options=self.question_options,
+        )
+        return question_html_content
 
     ################
     # Helpful methods
     ################
     @classmethod
-    def example(cls) -> QuestionBudget:
+    def example(cls, include_comment: bool = True) -> QuestionBudget:
+        """Return an example of a budget question."""
         return cls(
             question_name="food_budget",
             question_text="How would you allocate $100?",
             question_options=["Pizza", "Ice Cream", "Burgers", "Salad"],
             budget_sum=100,
+            include_comment=include_comment,
         )
 
 
 def main():
+    """Create an example of a budget question and demonstrate its functionality."""
     from edsl.questions.QuestionBudget import QuestionBudget
 
     q = QuestionBudget.example()
     q.question_text
     q.question_options
     q.question_name
-    q.short_names_dict
     # validate an answer
-    q.validate_answer(
+    q._validate_answer(
         {"answer": {0: 100, 1: 0, 2: 0, 3: 0}, "comment": "I like custard"}
     )
     # translate answer code
-    q.translate_answer_code_to_answer({0: 100, 1: 0, 2: 0, 3: 0})
+    q._translate_answer_code_to_answer({0: 100, 1: 0, 2: 0, 3: 0})
     # simulate answer
-    q.simulate_answer()
-    q.simulate_answer(human_readable=False)
-    q.validate_answer(q.simulate_answer(human_readable=False))
+    q._simulate_answer()
+    q._simulate_answer(human_readable=False)
+    q._validate_answer(q._simulate_answer(human_readable=False))
     # serialization (inherits from Question)
     q.to_dict()
     assert q.from_dict(q.to_dict()) == q
 
 
 if __name__ == "__main__":
-    q = QuestionBudget.example()
-    results = q.run()
+    # q = QuestionBudget.example()
+    # results = q.run()
+
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)