edsl 0.1.15__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/questions/SimpleAskMixin.py

@@ -0,0 +1,74 @@
+import math
+
+
+def logprob_to_prob(logprob):
+    if logprob is None:
+        return None
+    return math.exp(logprob)
+
+
+def format_output(data):
+    from rich.table import Table
+    from rich.console import Console
+
+    content = data["choices"][0]["logprobs"]["content"]
+    table = Table(show_header=True, header_style="bold magenta")
+
+    # First pass to determine the maximum number of top tokens
+    max_tokens = max(len(item["top_logprobs"]) for item in content)
+
+    # Set up the table columns
+    table.add_column("Token", style="bold")
+    for i in range(max_tokens):
+        table.add_column(f"Top Token {i+1}")
+
+    for item in content:
+        token = item["token"].strip()
+        top_tokens = [
+            (top["token"].strip(), logprob_to_prob(top.get("logprob")))
+            for top in item["top_logprobs"]
+        ]
+
+        # Row data starts with the main token
+        row = [token]
+
+        # Add each top token with coloring based on probability
+        for token, probability in top_tokens:
+            if probability is not None:
+                # Assign color based on probability value
+                if probability > 0.5:
+                    color = "green"
+                elif probability > 0.2:
+                    color = "yellow"
+                else:
+                    color = "red"
+                formatted_token = f"[{color}]{token} ({probability:.2f})[/]"
+            else:
+                formatted_token = f"{token} (N/A)"
+            row.append(formatted_token)
+
+        # Ensure row has enough columns
+        while len(row) < max_tokens + 1:
+            row.append("")
+
+        table.add_row(*row)
+
+    console = Console()
+    console.print(table)
+
+
+class SimpleAskMixin:
+    # def simple_ask(self, question: QuestionBase, system_prompt = "You are a helpful agent pretending to be a human.", top_logprobs = 2):
+
+    def simple_ask(
+        self,
+        model=None,
+        system_prompt="You are a helpful agent pretending to be a human. Do not break character",
+        top_logprobs=4,
+    ):
+        from edsl.language_models.model import Model
+
+        if model is None:
+            model = Model()
+        response = model.simple_ask(self, system_prompt, top_logprobs)
+        return format_output(response)

edsl/questions/__init__.py

@@ -1,25 +1,28 @@
 # Schemas
 from edsl.questions.settings import Settings
+from edsl.questions.register_questions_meta import RegisterQuestionsMeta
 
 # Base Class
-from edsl.questions.Question import Question
+from edsl.questions.QuestionBase import QuestionBase
 
 # Core Questions
-from edsl.questions.QuestionBudget import QuestionBudget
 from edsl.questions.QuestionCheckBox import QuestionCheckBox
 from edsl.questions.QuestionExtract import QuestionExtract
 from edsl.questions.QuestionFreeText import QuestionFreeText
 from edsl.questions.QuestionFunctional import QuestionFunctional
 from edsl.questions.QuestionList import QuestionList
+from edsl.questions.QuestionMatrix import QuestionMatrix
+from edsl.questions.QuestionDict import QuestionDict
 from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
 from edsl.questions.QuestionNumerical import QuestionNumerical
+from edsl.questions.QuestionBudget import QuestionBudget
 from edsl.questions.QuestionRank import QuestionRank
 
-# Questions derived from core questions
+# # # Questions derived from core questions
 from edsl.questions.derived.QuestionLikertFive import QuestionLikertFive
 from edsl.questions.derived.QuestionLinearScale import QuestionLinearScale
-from edsl.questions.derived.QuestionTopK import QuestionTopK
 from edsl.questions.derived.QuestionYesNo import QuestionYesNo
+from edsl.questions.derived.QuestionTopK import QuestionTopK
 
-# Compose Questions
-from edsl.questions.compose_questions import compose_questions
+# # Compose Questions
+# from edsl.questions.compose_questions import compose_questions

edsl/questions/{AnswerValidatorMixin.py → answer_validator_mixin.py}

@@ -1,24 +1,42 @@
+"""Mixin with validators for LLM answers to questions."""
+
 import re
 from typing import Any, Type, Union
-from edsl.exceptions import (
+from edsl.exceptions.questions import (
     QuestionAnswerValidationError,
 )
 
 
 class AnswerValidatorMixin:
     """
-    Mixin with validators for LLM answers to questions
+    Mixin with validators for LLM answers to questions.
+
     - Template validation: validators for the entire answer object format
     - Value validation: validators for specific values
     - Question specific validation: validators for specific question types
     """
 
+    def failing_job(self):
+        from edsl import Agent
+
+        a = Agent()
+
+        def f(self, question, scenario):
+            return []
+
+        a.add_direct_question_answering_method(f, validate_response=True)
+        from edsl import QuestionNumerical
+
+        q = QuestionNumerical.example()
+        results = q.by(a).run()
+        return results
+
     #####################
     # TEMPLATE VALIDATION
     #####################
-    def validate_answer_template_basic(self, answer: Any) -> None:
-        """
-        Checks that the answer (i) is a dictionary (ii) has an 'answer' key
+    def _validate_answer_template_basic(self, answer: Any) -> None:
+        """Check that the answer (i) is a dictionary (ii) has an 'answer' key.
+
         - E.g., both {'answer': 1} and {'answer': {'a': 1}, 'other_key'=[1,2,3]} are valid
         """
         if not isinstance(answer, dict):
@@ -33,21 +51,25 @@ class AnswerValidatorMixin:
     #####################
     # VALUE VALIDATION
     #####################
-    def validate_answer_key_value(
+    def _validate_answer_key_value(
         self, answer: dict[str, Any], key: str, of_type: Type
     ) -> None:
-        """Checks that the value of a key is of the specified type"""
+        """Check that the value of a key is of the specified type."""
         if not isinstance(answer.get(key), of_type):
             raise QuestionAnswerValidationError(
                 f"""Answer key '{key}' must be of type {of_type.__name__};
                 (got {answer.get(key)}) which is of type {type(answer.get(key))}."""
             )
 
-    def validate_answer_key_value_numeric(
+    def _validate_answer_key_value_numeric(
         self, answer: dict[str, Any], key: str
     ) -> None:
-        """Checks that the value of a key is numeric (int or float)"""
+        """Check that the value is numeric (int or float).
+        Can also deal with strings that contain commas and other characters.
+
+        """
         value = answer.get(key)
+        initial_value = value
         if type(value) == str:
             value = value.replace(",", "")
             value = "".join(re.findall(r"[-+]?\d*\.\d+|\d+", value))
@@ -59,7 +81,7 @@ class AnswerValidatorMixin:
                     value = float(value)
                 except ValueError:
                     raise QuestionAnswerValidationError(
-                        f"Answer should be numerical (int or float)."
+                        f"Answer should be numerical (int or float). Got '{initial_value}'"
                     )
             return None
         elif type(value) == int or type(value) == float:
@@ -72,9 +94,10 @@ class AnswerValidatorMixin:
     #####################
     # QUESTION SPECIFIC VALIDATION
     #####################
-    def validate_answer_budget(self, answer: dict[str, Any]) -> None:
-        """
-        QuestionBudget-specific answer validation. Checks that answer["answer"]:
+    def _validate_answer_budget(self, answer: dict[str, Any]) -> None:
+        """Validate QuestionBudget-specific answer.
+
+        Check that answer["answer"]:
         - has keys that are in the range of the number of options
         - has values that are non-negative integers
         - has values that sum to `budget_sum`
@@ -95,17 +118,21 @@ class AnswerValidatorMixin:
             )
         if any([int(key) not in acceptable_answer_keys for key in answer.keys()]):
             raise QuestionAnswerValidationError(
-                f"Budget keys must be in {acceptable_answer_keys}, but got {answer_keys}"
+                f"Budget keys must be in {acceptable_answer_keys}, but got {answer_keys}."
             )
         if acceptable_answer_keys != answer_keys:
             missing_keys = acceptable_answer_keys - answer_keys
             raise QuestionAnswerValidationError(
-                f"All but keys must be represented in the answer. Missing: {missing_keys}"
+                f"All but keys must be represented in the answer. Missing: {missing_keys}."
             )
 
-    def validate_answer_checkbox(self, answer: dict[str, Union[str, int]]) -> None:
-        """
-        QuestionCheckbox-specific answer validation. Checks that answer["answer"]:
+    def _validate_answer_checkbox(self, answer: dict[str, Union[str, int]]) -> None:
+        """Validate QuestionCheckbox-specific answer.
+
+        :param answer: Answer to validate
+
+
+        Check that answer["answer"]:
         - has elements that are strings, bytes-like objects or real numbers evaluating to integers
         - has elements that are in the range of the number of options
         - has at least `min_selections` elements, if provided
@@ -126,16 +153,17 @@ class AnswerValidatorMixin:
                 )
         if self.min_selections is not None and len(answer_codes) < self.min_selections:
             raise QuestionAnswerValidationError(
-                f"Answer {answer_codes} has fewer than {self.min_selections} options selected."
+                f"Answer codes, {answer_codes}, has fewer than {self.min_selections} options selected."
             )
         if self.max_selections is not None and len(answer_codes) > self.max_selections:
             raise QuestionAnswerValidationError(
-                f"Answer {answer_codes} has more than {self.max_selections} options selected."
+                f"Answer codes, {answer_codes}, has more than {self.max_selections} options selected."
             )
 
-    def validate_answer_extract(self, answer: dict[str, Any]) -> None:
-        """
-        QuestionExtract-specific answer validation. Checks that answer["answer"]:
+    def _validate_answer_extract(self, answer: dict[str, Any]) -> None:
+        """Validate QuestionExtract-specific answer.
+
+        Check that answer["answer"]:
         - does not have keys that are not in the answer template
         - has all keys that are in the answer template
         """
@@ -143,16 +171,17 @@ class AnswerValidatorMixin:
         acceptable_answer_keys = set(self.answer_template.keys())
         if any([key not in acceptable_answer_keys for key in value.keys()]):
             raise QuestionAnswerValidationError(
-                f"Answer keys must be in {acceptable_answer_keys}, but got {value.keys()}"
+                f"Answer keys must be in {acceptable_answer_keys}, but got {value.keys()}."
             )
         if any([key not in value.keys() for key in acceptable_answer_keys]):
             raise QuestionAnswerValidationError(
-                f"Answer must have all keys in {acceptable_answer_keys}, but got {value.keys()}"
+                f"Answer must have all keys in {acceptable_answer_keys}, but got {value.keys()}."
             )
 
-    def validate_answer_list(self, answer: dict[str, Union[list, str]]) -> None:
-        """
-        QuestionList-specific answer validation. Checks that answer["answer"]:
+    def _validate_answer_list(self, answer: dict[str, Union[list, str]]) -> None:
+        """Validate QuestionList-specific answer.
+
+        Check that answer["answer"]:
         - is not empty, if `allow_nonresponse` is False
         - has no more than `max_list_items` elements
         - has no empty strings
@@ -177,13 +206,19 @@ class AnswerValidatorMixin:
                 f"Answer cannot contain empty strings, but got {value}."
             )
 
-    def validate_answer_numerical(self, answer: dict) -> None:
-        """
-        QuestionNumerical-specific answer validation. Checks that answer["answer"]:
+    def _validate_answer_numerical(self, answer: dict) -> None:
+        """Validate QuestionNumerical-specific answer.
+
+        Check that answer["answer"]:
         - is not less than `min_value`
         - is not greater than `max_value`
         """
-        value = float(answer.get("answer"))
+        try:
+            value = float(answer.get("answer"))
+        except ValueError:
+            raise QuestionAnswerValidationError(
+                f"Answer must real number or convertible to a real number (got {answer.get('answer')})."
+            )
         if self.min_value is not None and value < self.min_value:
             raise QuestionAnswerValidationError(
                 f"Value {value} is less than {self.min_value}"
@@ -194,11 +229,12 @@ class AnswerValidatorMixin:
             )
         return value
 
-    def validate_answer_multiple_choice(
+    def _validate_answer_multiple_choice(
         self, answer: dict[str, Union[str, int]]
     ) -> None:
-        """
-        QuestionMultipleChoice-specific answer validation. Checks that answer["answer"]:
+        """Validate QuestionMultipleChoice-specific answer.
+
+        Check that answer["answer"]:
         - is a string, bytes-like object or real number
         - is a non-negative integer
         - is in the range of the number of options
@@ -218,9 +254,10 @@ class AnswerValidatorMixin:
                 f"Answer code {value} must be in {list(range(len(self.question_options)))}."
             )
 
-    def validate_answer_rank(self, answer: dict[str, Union[str, int]]) -> None:
-        """
-        QuestionRank-specific answer validation. Checks that answer["answer"]:
+    def _validate_answer_rank(self, answer: dict[str, Union[str, int]]) -> None:
+        """Validate QuestionRank-specific answer.
+
+        Check that answer["answer"]:
         - contains only integers
         - contains only integers in the range of the number of options
         - has the correct number of elements
@@ -246,3 +283,81 @@ class AnswerValidatorMixin:
             raise QuestionAnswerValidationError(
                 f"Rank answer {value}, but exactly {self.num_selections} selections required."
             )
+
+    def _validate_answer_matrix(self, answer: dict[str, Any]) -> None:
+        """Validate QuestionMatrix-specific answer.
+
+        Check that answer["answer"]:
+        - is a dictionary
+        - has all required question_items as keys
+        - has values that are valid options from question_options
+        - has the correct number of responses (one per item)
+        """
+        value = answer.get("answer")
+
+        # Check that answer is a dictionary
+        if not isinstance(value, dict):
+            raise QuestionAnswerValidationError(
+                f"Matrix answer must be a dictionary mapping items to responses (got {value})"
+            )
+
+        # Check that all required items are present
+        required_items = set(self.question_items)
+        provided_items = set(value.keys())
+
+        if missing_items := (required_items - provided_items):
+            raise QuestionAnswerValidationError(
+                f"Missing responses for items: {missing_items}"
+            )
+
+        if extra_items := (provided_items - required_items):
+            raise QuestionAnswerValidationError(
+                f"Unexpected responses for items: {extra_items}"
+            )
+
+        # Check that all responses are valid options
+        valid_options = set(self.question_options)
+        for item, response in value.items():
+            if response not in valid_options:
+                raise QuestionAnswerValidationError(
+                    f"Invalid response '{response}' for item '{item}'. "
+                    f"Must be one of: {valid_options}"
+                )
+
+    def _validate_answer_dict(self, answer: dict[str, Any]) -> None:
+        """Validate QuestionDict-specific answer.
+
+        Check that answer["answer"]:
+        - is a dictionary
+        - has all required answer_keys as keys
+        """
+        value = answer.get("answer")
+
+        # Check that answer is a dictionary
+        if not isinstance(value, dict):
+            raise QuestionAnswerValidationError(
+                f"Dict answer must be a dictionary mapping values to specified keys (got {value})"
+            )
+
+        # Check that all required answer keys are present
+        required_keys = set(self.answer_keys)
+        provided_keys = set(value.keys())
+
+        if missing_keys := (required_keys - provided_keys):
+            raise QuestionAnswerValidationError(
+                f"Missing required keys: {missing_keys}"
+            )
+
+        if extra_keys := (provided_keys - required_keys):
+            raise QuestionAnswerValidationError(
+                f"Unexpected keys: {extra_keys}"
+            )
+
+
+if __name__ == "__main__":
+    pass
+    # import doctest
+
+    # doctest.testmod(optionflags=doctest.ELLIPSIS)
+
+    # results = AnswerValidatorMixin().failing_job()

edsl/questions/compose_questions.py

@@ -1,17 +1,20 @@
-from edsl.questions import Question, QuestionFunctional
-from edsl.agents import Agent
+"""Compose two questions where the answer to q1 is used as an input to q2."""
+
+from edsl.questions import QuestionFunctional
+from edsl.questions.QuestionBase import QuestionBase
 from edsl.scenarios.Scenario import Scenario
 
 
 def compose_questions(
-    q1: Question, q2: Question, question_name: str = None
+    q1: QuestionBase, q2: QuestionBase, question_name: str = None
 ) -> QuestionFunctional:
     """
-    Composes two questions. The answer to q1 is used as the input to q2.
+    Compose two questions where the answer to q1 is used as an input to q2.
+
     The resulting question is a question that can be used like other questions.
-    Note:
-    - One way the same result can be achieved is through q1+q2.
-      Please see the __add__ method in Question.
+    Note that the same result can also be achieved in other ways:
+    - Using the `add_targeted_memory(q2, q1)` method in Survey
+    - Using the __add__ method in Question
     """
     if question_name is None:
         question_name = f"{q1.question_name}_{q2.question_name}"
@@ -24,7 +27,10 @@ def compose_questions(
     def combo(
         scenario: Scenario, agent_traits: dict[str, str] = None
     ) -> QuestionFunctional:
+        """Return the answer to the second question given the answer to the first question."""
         # get the answer to the first question
+        from edsl.agents.Agent import Agent
+
         first_answer = (
             q1.by(scenario)
             .by(Agent(traits=agent_traits))

edsl/questions/data_structures.py

@@ -0,0 +1,20 @@
+from typing import Any, Optional, TypedDict
+from pydantic import BaseModel
+
+
+class RawEdslAnswerDict(TypedDict):
+    answer: Any
+    comment: Optional[str]
+    generated_tokens: Optional[str]
+
+
+class BaseResponse(BaseModel):
+    answer: Any
+    comment: Optional[str] = None
+    generated_tokens: Optional[str] = None
+
+
+class EdslAnswerDict(TypedDict):
+    answer: Any
+    comment: Optional[str]
+    generated_tokens: Optional[str]

edsl/questions/decorators.py

@@ -0,0 +1,21 @@
+from typing import Optional, Callable, TypeVar
+
+T = TypeVar("T")
+
+
+def inject_exception(func: Callable[..., T]) -> Callable[..., T]:
+    def wrapper(
+        cls,
+        exception_to_throw: Optional[Exception] = None,
+        override_answer: Optional[dict] = None,
+        *args,
+        **kwargs
+    ) -> T:
+        base_instance = func(cls, *args, **kwargs)
+        if exception_to_throw:
+            base_instance.exception_to_throw = exception_to_throw
+        if override_answer:
+            base_instance.override_answer = override_answer
+        return base_instance
+
+    return wrapper

edsl/questions/derived/QuestionLikertFive.py

@@ -2,22 +2,11 @@ from __future__ import annotations
 from typing import Optional
 from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice
 
+from edsl.questions.decorators import inject_exception
 
-class QuestionLikertFive(QuestionMultipleChoice):
-    """
-    This question asks the user to respond to a statement on a 5-point Likert scale.
-
-    Arguments:
-    - `question_name` is the name of the question (string)
-    - `question_text` is the text of the question (string)
 
-    Optional arguments:
-    - `instructions` are the instructions for the question (string). If not provided, the default instructions are used. To view them, run `QuestionLikertFive.default_instructions`
-    - `question_options` are the options the user should select from (list of strings). If not provided, the default likert options are used. To view them, run `QuestionLikertFive.likert_options`
-    - `short_names_dict` maps question_options to short names (dictionary mapping strings to strings)
-
-    For an example, see `QuestionLikertFive.example()`
-    """
+class QuestionLikertFive(QuestionMultipleChoice):
+    """This question prompts the agent to respond to a statement on a 5-point Likert scale."""
 
     question_type = "likert_five"
     likert_options: list[str] = [
@@ -27,27 +16,36 @@ class QuestionLikertFive(QuestionMultipleChoice):
         "Agree",
         "Strongly agree",
     ]
-    # default_instructions = QuestionMultipleChoice.default_instructions
 
     def __init__(
         self,
         question_name: str,
         question_text: str,
         question_options: Optional[list[str]] = likert_options,
-        short_names_dict: Optional[dict[str, str]] = None,
+        answering_instructions: Optional[str] = None,
+        question_presentation: Optional[str] = None,
+        include_comment: bool = True,
     ):
+        """Initialize the question.
+
+        :param question_name: The name of the question.
+        :param question_text: The text of the question.
+        :param question_options: The options the respondent should select from (list of strings). If not provided, the default Likert options are used (['Strongly disagree', 'Disagree', 'Neutral', 'Agree', 'Strongly agree']). To view them, run `QuestionLikertFive.likert_options`.
+        """
         super().__init__(
             question_name=question_name,
             question_text=question_text,
             question_options=question_options,
-            short_names_dict=short_names_dict,
+            use_code=False,
+            include_comment=include_comment,
+            answering_instructions=answering_instructions,
+            question_presentation=question_presentation,
         )
 
-    ################
-    # Helpful
-    ################
     @classmethod
+    @inject_exception
     def example(cls) -> QuestionLikertFive:
+        """Return an example question."""
         return cls(
             question_name="happy_raining",
             question_text="I'm only happy when it rains.",
@@ -55,20 +53,24 @@ class QuestionLikertFive(QuestionMultipleChoice):
 
 
 def main():
+    """Test QuestionLikertFive."""
     from edsl.questions.derived.QuestionLikertFive import QuestionLikertFive
 
     q = QuestionLikertFive.example()
     q.question_text
     q.question_options
     q.question_name
-    q.short_names_dict
     # validate an answer
-    q.validate_answer({"answer": 0, "comment": "I like custard"})
+    q._validate_answer({"answer": 0, "comment": "I like custard"})
     # translate answer code
-    q.translate_answer_code_to_answer(0, {})
-    q.simulate_answer()
-    q.simulate_answer(human_readable=False)
-    q.validate_answer(q.simulate_answer(human_readable=False))
+    q._translate_answer_code_to_answer(0, {})
+    q._simulate_answer()
+    q._simulate_answer(human_readable=False)
+    q._validate_answer(q._simulate_answer(human_readable=False))
     # serialization (inherits from Question)
     q.to_dict()
     assert q.from_dict(q.to_dict()) == q
+
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)