edsl 0.1.31.dev4__py3-none-any.whl → 0.1.33__py3-none-any.whl

edsl/questions/QuestionBaseGenMixin.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+import copy
+import itertools
+from typing import Optional, List, Callable, Type
+from typing import TypeVar
+
+
+class QuestionBaseGenMixin:
+    def copy(self) -> QuestionBase:
+        """Return a deep copy of the question.
+
+        >>> from edsl.questions import QuestionFreeText
+        >>> q = QuestionFreeText(question_name = "color", question_text = "What is your favorite color?")
+        >>> q2 = q.copy()
+        >>> q2.question_name
+        'color'
+
+        """
+        return copy.deepcopy(self)
+
+    def option_permutations(self) -> list[QuestionBase]:
+        """Return a list of questions with all possible permutations of the options.
+
+        >>> from edsl import QuestionMultipleChoice as Q
+        >>> len(Q.example().option_permutations())
+        24
+        """
+
+        if not hasattr(self, "question_options"):
+            return [self]
+
+        questions = []
+        for index, permutation in enumerate(
+            itertools.permutations(self.question_options)
+        ):
+            question = copy.deepcopy(self)
+            question.question_options = list(permutation)
+            question.question_name = f"{self.question_name}_{index}"
+            questions.append(question)
+        return questions
+
+    def loop(self, scenario_list: ScenarioList) -> List[QuestionBase]:
+        """Return a list of questions with the question name modified for each scenario.
+
+        :param scenario_list: The list of scenarios to loop through.
+
+        >>> from edsl import QuestionFreeText
+        >>> from edsl import ScenarioList
+        >>> q = QuestionFreeText(question_text = "What are your thoughts on: {{ subject}}?", question_name = "base_{{subject}}")
+        >>> len(q.loop(ScenarioList.from_list("subject", ["Math", "Economics", "Chemistry"])))
+        3
+
+        """
+        from jinja2 import Environment
+        from edsl.questions.QuestionBase import QuestionBase
+
+        starting_name = self.question_name
+        questions = []
+        for index, scenario in enumerate(scenario_list):
+            env = Environment()
+            new_data = self.to_dict().copy()
+            for key, value in [(k, v) for k, v in new_data.items() if v is not None]:
+                if (
+                    isinstance(value, str) or isinstance(value, int)
+                ) and key != "question_options":
+                    new_data[key] = env.from_string(value).render(scenario)
+                elif isinstance(value, list):
+                    new_data[key] = [
+                        env.from_string(v).render(scenario) if isinstance(v, str) else v
+                        for v in value
+                    ]
+                elif isinstance(value, dict):
+                    new_data[key] = {
+                        (
+                            env.from_string(k).render(scenario)
+                            if isinstance(k, str)
+                            else k
+                        ): (
+                            env.from_string(v).render(scenario)
+                            if isinstance(v, str)
+                            else v
+                        )
+                        for k, v in value.items()
+                    }
+                elif key == "question_options" and isinstance(value, str):
+                    new_data[key] = value
+                else:
+                    raise ValueError(
+                        f"Unexpected value type: {type(value)} for key '{key}'"
+                    )
+
+            if new_data["question_name"] == starting_name:
+                new_data["question_name"] = new_data["question_name"] + f"_{index}"
+
+            questions.append(QuestionBase.from_dict(new_data))
+        return questions
+
+    def apply_function(self, func: Callable, exclude_components=None) -> QuestionBase:
+        """Apply a function to the question parts
+
+        >>> from edsl.questions import QuestionFreeText
+        >>> q = QuestionFreeText(question_name = "color", question_text = "What is your favorite color?")
+        >>> shouting = lambda x: x.upper()
+        >>> q.apply_function(shouting)
+        Question('free_text', question_name = \"""color\""", question_text = \"""WHAT IS YOUR FAVORITE COLOR?\""")
+
+        """
+        from edsl.questions.QuestionBase import QuestionBase
+
+        if exclude_components is None:
+            exclude_components = ["question_name", "question_type"]
+
+        d = copy.deepcopy(self._to_dict())
+        for key, value in d.items():
+            if key in exclude_components:
+                continue
+            if isinstance(value, dict):
+                for k, v in value.items():
+                    value[k] = func(v)
+                d[key] = value
+                continue
+            if isinstance(value, list):
+                value = [func(v) for v in value]
+                d[key] = value
+                continue
+            d[key] = func(value)
+        return QuestionBase.from_dict(d)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

edsl/questions/QuestionBasePromptsMixin.py

@@ -0,0 +1,266 @@
+from importlib import resources
+from typing import Optional
+from edsl.prompts import Prompt
+from edsl.exceptions.questions import QuestionAnswerValidationError
+
+from functools import lru_cache
+
+
+class TemplateManager:
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance._template_cache = {}
+        return cls._instance
+
+    @lru_cache(maxsize=None)
+    def get_template(self, question_type, template_name):
+        if (question_type, template_name) not in self._template_cache:
+            with resources.open_text(
+                f"edsl.questions.templates.{question_type}", template_name
+            ) as file:
+                self._template_cache[(question_type, template_name)] = file.read()
+        return self._template_cache[(question_type, template_name)]
+
+
+# Global instance
+template_manager = TemplateManager()
+
+
+class QuestionBasePromptsMixin:
+    # @classmethod
+    # @lru_cache(maxsize=1)
+    # def _read_template(cls, template_name):
+    #     with resources.open_text(
+    #         f"edsl.questions.templates.{cls.question_type}", template_name
+    #     ) as file:
+    #         return file.read()
+
+    @classmethod
+    def applicable_prompts(
+        cls, model: Optional[str] = None
+    ) -> list[type["PromptBase"]]:
+        """Get the prompts that are applicable to the question type.
+
+        :param model: The language model to use.
+
+        >>> from edsl.questions import QuestionFreeText
+        >>> QuestionFreeText.applicable_prompts()
+        [<class 'edsl.prompts.library.question_freetext.FreeText'>]
+
+        :param model: The language model to use. If None, assumes does not matter.
+
+        """
+        from edsl.prompts.registry import get_classes as prompt_lookup
+
+        applicable_prompts = prompt_lookup(
+            component_type="question_instructions",
+            question_type=cls.question_type,
+            model=model,
+        )
+        return applicable_prompts
+
+    @property
+    def model_instructions(self) -> dict:
+        """Get the model-specific instructions for the question."""
+        if not hasattr(self, "_model_instructions"):
+            self._model_instructions = {}
+        return self._model_instructions
+
+    def _all_text(self) -> str:
+        """Return the question text.
+
+        >>> from edsl import QuestionMultipleChoice as Q
+        >>> Q.example()._all_text()
+        "how_feelingHow are you?['Good', 'Great', 'OK', 'Bad']"
+        """
+        txt = ""
+        for key, value in self.data.items():
+            if isinstance(value, str):
+                txt += value
+            elif isinstance(value, list):
+                txt += "".join(str(value))
+        return txt
+
+    @model_instructions.setter
+    def model_instructions(self, data: dict):
+        """Set the model-specific instructions for the question."""
+        self._model_instructions = data
+
+    def add_model_instructions(
+        self, *, instructions: str, model: Optional[str] = None
+    ) -> None:
+        """Add model-specific instructions for the question that override the default instructions.
+
+        :param instructions: The instructions to add. This is typically a jinja2 template.
+        :param model: The language model for this instruction.
+
+        >>> from edsl.questions import QuestionFreeText
+        >>> q = QuestionFreeText(question_name = "color", question_text = "What is your favorite color?")
+        >>> q.add_model_instructions(instructions = "{{question_text}}. Answer in valid JSON like so {'answer': 'comment: <>}", model = "gpt3")
+        >>> q.get_instructions(model = "gpt3")
+        Prompt(text=\"""{{question_text}}. Answer in valid JSON like so {'answer': 'comment: <>}\""")
+        """
+        from edsl import Model
+
+        if not hasattr(self, "_model_instructions"):
+            self._model_instructions = {}
+        if model is None:
+            # if not model is passed, all the models are mapped to this instruction, including 'None'
+            self._model_instructions = {
+                model_name: instructions
+                for model_name in Model.available(name_only=True)
+            }
+            self._model_instructions.update({model: instructions})
+        else:
+            self._model_instructions.update({model: instructions})
+
+    @classmethod
+    def path_to_folder(cls) -> str:
+        return resources.files(f"edsl.questions.templates", cls.question_type)
+
+    @property
+    def response_model(self) -> type["BaseModel"]:
+        if self._response_model is not None:
+            return self._response_model
+        else:
+            return self.create_response_model()
+
+    @property
+    def use_code(self) -> bool:
+        if hasattr(self, "_use_code"):
+            return self._use_code
+        return True
+
+    @use_code.setter
+    def use_code(self, value: bool) -> None:
+        self._use_code = value
+
+    @property
+    def include_comment(self) -> bool:
+        if hasattr(self, "_include_comment"):
+            return self._include_comment
+        return True
+
+    @include_comment.setter
+    def include_comment(self, value: bool) -> None:
+        self._include_comment = value
+
+    @classmethod
+    def default_answering_instructions(cls) -> str:
+        # template_text = cls._read_template("answering_instructions.jinja")
+        template_text = template_manager.get_template(
+            cls.question_type, "answering_instructions.jinja"
+        )
+        return Prompt(text=template_text)
+
+    @classmethod
+    def default_question_presentation(cls):
+        # template_text = cls._read_template("question_presentation.jinja")
+        template_text = template_manager.get_template(
+            cls.question_type, "question_presentation.jinja"
+        )
+        return Prompt(text=template_text)
+
+    @property
+    def answering_instructions(self) -> str:
+        if self._answering_instructions is None:
+            return self.default_answering_instructions()
+        return self._answering_instructions
+
+    @answering_instructions.setter
+    def answering_instructions(self, value) -> None:
+        self._answering_instructions = value
+
+    # @classmethod
+    # def default_answering_instructions(cls) -> str:
+    #     with resources.open_text(
+    #         f"edsl.questions.templates.{cls.question_type}",
+    #         "answering_instructions.jinja",
+    #     ) as file:
+    #         return Prompt(text=file.read())
+
+    # @classmethod
+    # def default_question_presentation(cls):
+    #     with resources.open_text(
+    #         f"edsl.questions.templates.{cls.question_type}",
+    #         "question_presentation.jinja",
+    #     ) as file:
+    #         return Prompt(text=file.read())
+
+    @property
+    def question_presentation(self):
+        if self._question_presentation is None:
+            return self.default_question_presentation()
+        return self._question_presentation
+
+    @question_presentation.setter
+    def question_presentation(self, value):
+        self._question_presentation = value
+
+    def prompt_preview(self, scenario=None, agent=None):
+        return self.new_default_instructions.render(
+            self.data
+            | {
+                "include_comment": getattr(self, "_include_comment", True),
+                "use_code": getattr(self, "_use_code", True),
+            }
+            | ({"scenario": scenario} or {})
+            | ({"agent": agent} or {})
+        )
+
+    @classmethod
+    def self_check(cls):
+        q = cls.example()
+        for answer, params in q.response_validator.valid_examples:
+            for key, value in params.items():
+                setattr(q, key, value)
+            q._validate_answer(answer)
+        for answer, params, reason in q.response_validator.invalid_examples:
+            for key, value in params.items():
+                setattr(q, key, value)
+            try:
+                q._validate_answer(answer)
+            except QuestionAnswerValidationError:
+                pass
+            else:
+                raise ValueError(f"Example {answer} should have failed for {reason}.")
+
+    @property
+    def new_default_instructions(self) -> "Prompt":
+        "This is set up as a property because there are mutable question values that determine how it is rendered."
+        return self.question_presentation + self.answering_instructions
+
+    @property
+    def parameters(self) -> set[str]:
+        """Return the parameters of the question."""
+        from jinja2 import Environment, meta
+
+        env = Environment()
+        # Parse the template
+        txt = self._all_text()
+        # txt = self.question_text
+        # if hasattr(self, "question_options"):
+        #    txt += " ".join(self.question_options)
+        parsed_content = env.parse(txt)
+        # Extract undeclared variables
+        variables = meta.find_undeclared_variables(parsed_content)
+        # Return as a list
+        return set(variables)
+
+    def get_instructions(self, model: Optional[str] = None) -> type["PromptBase"]:
+        """Get the mathcing question-answering instructions for the question.
+
+        :param model: The language model to use.
+        """
+        from edsl.prompts.Prompt import Prompt
+
+        if model in self.model_instructions:
+            return Prompt(text=self.model_instructions[model])
+        else:
+            if hasattr(self, "new_default_instructions"):
+                return self.new_default_instructions
+            else:
+                return self.applicable_prompts(model)[0]()

edsl/questions/QuestionBudget.py

@@ -1,8 +1,60 @@
 from __future__ import annotations
-import random
-from typing import Any, Optional, Union
+from typing import Any, Optional, Union, List
+
+from pydantic import Field, BaseModel, validator
+
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import IntegerDescriptor, QuestionOptionsDescriptor
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+
+
+class BudgewResponseValidator(ResponseValidatorABC):
+    valid_examples = []
+
+    invalid_examples = []
+
+    def fix(self, response, verbose=False):
+        if verbose:
+            print(f"Fixing list response: {response}")
+        answer = str(response.get("answer") or response.get("generated_tokens", ""))
+        if len(answer.split(",")) > 0:
+            return (
+                {"answer": answer.split(",")} | {"comment": response.get("comment")}
+                if "comment" in response
+                else {}
+            )
+
+
+def create_budget_model(
+    budget_sum: float, permissive: bool, question_options: List[str]
+):
+    class BudgetResponse(BaseModel):
+        answer: List[float] = Field(
+            ...,
+            description="List of non-negative numbers representing budget allocation",
+            min_items=len(question_options),
+            max_items=len(question_options),
+        )
+        comment: Optional[str] = None
+        generated_tokens: Optional[str] = None
+
+        @validator("answer")
+        def validate_answer(cls, v):
+            if len(v) != len(question_options):
+                raise ValueError(f"Must provide {len(question_options)} values")
+            if any(x < 0 for x in v):
+                raise ValueError("All values must be non-negative")
+            total = sum(v)
+            if not permissive and total != budget_sum:
+                raise ValueError(f"Sum of numbers must equal {budget_sum}")
+            elif permissive and total > budget_sum:
+                raise ValueError(f"Sum of numbers cannot exceed {budget_sum}")
+            return v
+
+        class Config:
+            extra = "forbid"
+
+    return BudgetResponse
 
 
 class QuestionBudget(QuestionBase):
@@ -11,6 +63,8 @@ class QuestionBudget(QuestionBase):
     question_type = "budget"
     budget_sum: int = IntegerDescriptor(none_allowed=False)
     question_options: list[str] = QuestionOptionsDescriptor(q_budget=True)
+    _response_model = None
+    response_validator_class = BudgewResponseValidator
 
     def __init__(
         self,
@@ -18,6 +72,10 @@ class QuestionBudget(QuestionBase):
         question_text: str,
         question_options: list[str],
         budget_sum: int,
+        include_comment: bool = True,
+        question_presentation: Optional[str] = None,
+        answering_instructions: Optional[str] = None,
+        permissive: bool = False,
     ):
         """Instantiate a new QuestionBudget.
 
@@ -25,26 +83,24 @@ class QuestionBudget(QuestionBase):
         :param question_text: The text of the question.
         :param question_options: The options for allocation of the budget sum.
         :param budget_sum: The total amount of the budget to be allocated among the options.
-        :param instructions: Instructions for the question. If not provided, the default instructions are used. To view them, run `QuestionBudget.default_instructions`.
         """
         self.question_name = question_name
         self.question_text = question_text
         self.question_options = question_options
         self.budget_sum = budget_sum
-
-    ################
-    # Answer methods
-    ################
-    def _validate_answer(self, answer: dict[str, Any]) -> dict[str, Union[int, str]]:
-        """Validate the answer."""
-        self._validate_answer_template_basic(answer)
-        self._validate_answer_key_value(answer, "answer", dict)
-        self._validate_answer_budget(answer)
-        return answer
+        self.question_presentation = question_presentation
+        self.answering_instructions = answering_instructions
+        self.permissive = permissive
+        self.include_comment = include_comment
+
+    def create_response_model(self):
+        return create_budget_model(
+            self.budget_sum, self.permissive, self.question_options
+        )
 
     def _translate_answer_code_to_answer(
-        self, answer_codes: dict[str, int], scenario: "Scenario" = None
-    ):
+        self, answer_code, combined_dict
+    ) -> list[dict]:
         """
         Translate the answer codes to the actual answers.
 
@@ -53,35 +109,35 @@ class QuestionBudget(QuestionBase):
         This code will translate that to "a".
         """
         translated_codes = []
-        for answer_code, response in answer_codes.items():
-            translated_codes.append({self.question_options[int(answer_code)]: response})
+        for answer_code, question_option in zip(answer_code, self.question_options):
+            translated_codes.append({question_option: answer_code})
 
         return translated_codes
 
-    def _simulate_answer(self, human_readable=True):
-        """Simulate a valid answer for debugging purposes (what the validator expects)."""
-        from edsl.utilities.utilities import random_string
-
-        if human_readable:
-            keys = self.question_options
-        else:
-            keys = range(len(self.question_options))
-        remaining_budget = self.budget_sum
-        values = []
-        for _ in range(len(self.question_options)):
-            if _ == len(self.question_options) - 1:
-                # Assign remaining budget to the last value
-                values.append(remaining_budget)
-            else:
-                # Generate a random value between 0 and remaining budget
-                value = random.randint(0, remaining_budget)
-                values.append(value)
-                remaining_budget -= value
-        answer = dict(zip(keys, values))
-        return {
-            "answer": answer,
-            "comment": random_string(),
-        }
+    # def _simulate_answer(self, human_readable=True):
+    #     """Simulate a valid answer for debugging purposes (what the validator expects)."""
+    #     from edsl.utilities.utilities import random_string
+
+    #     if human_readable:
+    #         keys = self.question_options
+    #     else:
+    #         keys = range(len(self.question_options))
+    #     remaining_budget = self.budget_sum
+    #     values = []
+    #     for _ in range(len(self.question_options)):
+    #         if _ == len(self.question_options) - 1:
+    #             # Assign remaining budget to the last value
+    #             values.append(remaining_budget)
+    #         else:
+    #             # Generate a random value between 0 and remaining budget
+    #             value = random.randint(0, remaining_budget)
+    #             values.append(value)
+    #             remaining_budget -= value
+    #     answer = dict(zip(keys, values))
+    #     return {
+    #         "answer": answer,
+    #         "comment": random_string(),
+    #     }
 
     @property
     def question_html_content(self) -> str:
@@ -128,13 +184,14 @@ class QuestionBudget(QuestionBase):
     # Helpful methods
     ################
     @classmethod
-    def example(cls) -> QuestionBudget:
+    def example(cls, include_comment: bool = True) -> QuestionBudget:
         """Return an example of a budget question."""
         return cls(
             question_name="food_budget",
             question_text="How would you allocate $100?",
             question_options=["Pizza", "Ice Cream", "Burgers", "Salad"],
             budget_sum=100,
+            include_comment=include_comment,
         )