edsl 0.1.15__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/surveys/descriptors.py

@@ -1,6 +1,10 @@
+"""This module contains the descriptors for the classes in the edsl package."""
+
 from abc import ABC, abstractmethod
-from typing import Any
-from edsl.questions import Question
+from typing import Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from edsl.questions.QuestionBase import QuestionBase
 
 
 class BaseDescriptor(ABC):
@@ -8,18 +12,20 @@ class BaseDescriptor(ABC):
 
     @abstractmethod
     def validate(self, value: Any) -> None:
-        """Validates the value. If it is invalid, raises an exception. If it is valid, does nothing."""
+        """Validate the value. If it is invalid, raise an exception. If it is valid, do nothing."""
         pass
 
     def __get__(self, instance, owner):
-        """"""
+        """Get the value of the attribute."""
         return instance.__dict__[self.name]
 
     def __set__(self, instance, value: Any) -> None:
+        """Set the value of the attribute."""
         self.validate(value, instance)
         instance.__dict__[self.name] = value
 
     def __set_name__(self, owner, name: str) -> None:
+        """Set the name of the attribute."""
         self.name = "_" + name
 
 
@@ -27,23 +33,28 @@ class QuestionsDescriptor(BaseDescriptor):
     """Descriptor for questions."""
 
     def __get__(self, instance, owner):
-        """"""
+        """Get the value of the attribute."""
         return instance.__dict__[self.name]
 
     def validate(self, value: Any, instance) -> None:
+        """Validate the value. If it is invalid, raise an exception. If it is valid, do nothing."""
+        from edsl.questions.QuestionBase import QuestionBase
+
         if not isinstance(value, list):
             raise TypeError("Questions must be a list.")
-        if not all(isinstance(question, Question) for question in value):
+        if not all(isinstance(question, QuestionBase) for question in value):
             raise TypeError("Questions must be a list of Question objects.")
         question_names = [question.question_name for question in value]
         if len(question_names) != len(set(question_names)):
             raise ValueError("Question names must be unique.")
 
     def __set__(self, instance, value: Any) -> None:
+        """Set the value of the attribute."""
         self.validate(value, instance)
         instance.__dict__[self.name] = []
         for question in value:
             instance.add_question(question)
 
     def __set_name__(self, owner, name: str) -> None:
+        """Set the name of the attribute."""
         self.name = "_" + name

edsl/surveys/instructions/ChangeInstruction.py

@@ -0,0 +1,48 @@
+from typing import List, Optional
+from edsl.utilities.remove_edsl_version import remove_edsl_version
+
+
+class ChangeInstruction:
+    def __init__(
+        self,
+        keep: Optional[List[str]] = None,
+        drop: Optional[List[str]] = None,
+    ):
+        if keep is None and drop is None:
+            raise ValueError("Keep and drop cannot both be None")
+
+        self.keep = keep or []
+        self.drop = drop or []
+
+    def include_instruction(self, instruction_name) -> bool:
+        return (instruction_name in self.keep) or (not instruction_name in self.drop)
+
+    def add_name(self, index) -> None:
+        self.name = "change_instruction_{}".format(index)
+
+    def __str__(self):
+        return self.text
+
+    def to_dict(self, add_edsl_version=True):
+        d = {
+            "keep": self.keep,
+            "drop": self.drop,
+        }
+        if add_edsl_version:
+            from edsl import __version__
+
+            d["edsl_version"] = __version__
+            d["edsl_class_name"] = "ChangeInstruction"
+
+        return d
+
+    def __hash__(self) -> int:
+        """Return a hash of the question."""
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self.to_dict(add_edsl_version=False))
+
+    @classmethod
+    @remove_edsl_version
+    def from_dict(cls, data):
+        return cls(data["keep"], data["drop"])

edsl/surveys/instructions/Instruction.py

@@ -0,0 +1,56 @@
+from typing import Union, Optional, List, Generator, Dict
+from edsl.utilities.remove_edsl_version import remove_edsl_version
+from edsl.Base import RepresentationMixin
+
+
+class Instruction(RepresentationMixin):
+    def __init__(
+        self, name, text, preamble="You were given the following instructions:"
+    ):
+        self.name = name
+        self.text = text
+        self.preamble = preamble
+
+    def __str__(self):
+        return self.text
+
+    def __repr__(self):
+        return """Instruction(name="{}", text="{}")""".format(self.name, self.text)
+
+    @classmethod
+    def example(cls) -> "Instruction":
+        return cls(name="example", text="This is an example instruction.")
+
+    def to_dict(self, add_edsl_version=True):
+        d = {
+            "name": self.name,
+            "text": self.text,
+            "edsl_class_name": "Instruction",
+            "preamble": self.preamble,
+        }
+        if add_edsl_version:
+            from edsl import __version__
+
+            d["edsl_version"] = __version__
+            d["edsl_class_name"] = "Instruction"
+        return d
+
+    def add_question(self, question) -> "Survey":
+        from edsl.surveys.Survey import Survey
+
+        return Survey([self, question])
+
+    def __hash__(self) -> int:
+        """Return a hash of the question."""
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self.to_dict(add_edsl_version=False))
+
+    @classmethod
+    @remove_edsl_version
+    def from_dict(cls, data):
+        return cls(
+            data["name"],
+            data["text"],
+            data.get("preamble", "You were given the following instructions:"),
+        )

edsl/surveys/instructions/InstructionCollection.py

@@ -0,0 +1,82 @@
+from typing import TYPE_CHECKING, Dict, List, Generator, Union
+from collections import UserDict
+
+from edsl.surveys.instructions.Instruction import Instruction
+from edsl.surveys.instructions.ChangeInstruction import ChangeInstruction
+
+if TYPE_CHECKING:
+    from edsl.questions.QuestionBase import QuestionBase
+
+
+class InstructionCollection(UserDict):
+    def __init__(
+        self,
+        instruction_names_to_instruction: Dict[str, Instruction],
+        questions: List["QuestionBase"],
+    ):
+        self.instruction_names_to_instruction = instruction_names_to_instruction
+        self.questions = questions
+        data = {}
+        for question in self.questions:
+            # just add both the question and the question name
+            data[question.name] = list(self.relevant_instructions(question))
+            # data[question] = list(self.relevant_instructions(question))
+        super().__init__(data)
+
+    def __getitem__(self, key):
+        # in case the person uses question instead of the name
+        from edsl.questions.QuestionBase import QuestionBase
+
+        if isinstance(key, QuestionBase):
+            key = key.name
+        return self.data[key]
+
+    @property
+    def question_names(self):
+        return [q.name for q in self.questions]
+
+    def question_index(self, question_name):
+        return self.question_names.index(question_name)
+
+    def _entries_before(
+        self, question_name
+    ) -> tuple[List[Instruction], List[ChangeInstruction]]:
+        if question_name not in self.question_names:
+            raise ValueError(
+                f"Question name not found in the list of questions: got '{question_name}'; list is {self.question_names}"
+            )
+        instructions, changes = [], []
+
+        index = self.question_index(question_name)
+        for instruction in self.instruction_names_to_instruction.values():
+            if instruction.pseudo_index < index:
+                if isinstance(instruction, Instruction):
+                    instructions.append(instruction)
+                elif isinstance(instruction, ChangeInstruction):
+                    changes.append(instruction)
+        return instructions, changes
+
+    def relevant_instructions(
+        self, question: Union[str, "QuestionBase"]
+    ) -> Generator[Instruction, None, None]:
+        ## Find all the questions that are after a given instruction
+        from edsl.questions.QuestionBase import QuestionBase
+
+        if isinstance(question, str):
+            question_name = question
+        elif isinstance(question, QuestionBase):
+            question_name = question.name
+
+        instructions_before, changes_before = self._entries_before(question_name)
+        keep_list = []
+        drop_list = []
+        for change in changes_before:
+            keep_list.extend(change.keep)
+            drop_list.extend(change.drop)
+
+        for instruction in instructions_before:
+            if instruction.name in keep_list or instruction.name not in drop_list:
+                yield instruction
+
+    def __len__(self):
+        return len(self.instruction_names_to_instruction)

edsl/templates/error_reporting/base.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Exception Details</title>
+    <style>
+    {{ css }}
+    </style>
+
+    <script>
+    {{ javascript }}
+    </script>
+
+</head>
+<body>
+    {% include 'overview.html' %}
+    {% include 'exceptions_by_type.html' %}
+    {% include 'exceptions_by_model.html' %}
+    {% include 'exceptions_by_question_name.html' %}
+    {% include 'interviews.html' %}
+    {% include 'performance_plot.html' %}
+</body>
+</html>

edsl/templates/error_reporting/exceptions_by_model.html

@@ -0,0 +1,35 @@
+<h2>Exceptions by Model</h2>
+<table border="1">
+    <thead>
+        <tr>
+            <th>Model</th>
+            <th>Number</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for model, exceptions in exceptions_by_model.items() %}
+            <tr>
+                <td>{{ model }}</td>
+                <td>{{ exceptions }}</td>
+            </tr>
+        {% endfor %}
+    </tbody>
+</table>
+
+<h2>Exceptions by Service</h2>
+<table border="1">
+    <thead>
+        <tr>
+            <th>Service</th>
+            <th>Number</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for service, exceptions in exceptions_by_service.items() %}
+            <tr>
+                <td>{{ service }}</td>
+                <td>{{ exceptions }}</td>
+            </tr>
+        {% endfor %}
+    </tbody>
+</table>

edsl/templates/error_reporting/exceptions_by_question_name.html

@@ -0,0 +1,17 @@
+<h2>Exceptions by Question Name</h2>
+<table border="1">
+    <thead>
+        <tr>
+            <th>Question Name</th>
+            <th>Number of Exceptions</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for question_name, exception_count in exceptions_by_question_name.items() %}
+            <tr>
+                <td>{{ question_name }}</td>
+                <td>{{ exception_count }}</td>
+            </tr>
+        {% endfor %}
+    </tbody>
+</table>

edsl/templates/error_reporting/exceptions_by_type.html

@@ -0,0 +1,17 @@
+<h2>Exceptions by Type</h2>
+<table border="1">
+    <thead>
+        <tr>
+            <th>Exception Type</th>
+            <th>Number</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for exception_type, exceptions in exceptions_by_type.items() %}
+            <tr>
+                <td>{{ exception_type }}</td>
+                <td>{{ exceptions }}</td>
+            </tr>
+        {% endfor %}
+    </tbody>
+</table>

edsl/templates/error_reporting/interview_details.html

@@ -0,0 +1,116 @@
+<div class="question">question_name: {{ question }}</div>
+
+
+<h2>Exception details</h2>
+
+{% for exception_message in exceptions %}
+<div class="exception-detail">
+        <div class="exception-header">
+        <span class="exception-exception">Exception: {{ exception_message.name }}</span>
+        <button class="toggle-btn">▼</button>
+        </div>
+        <div class="exception-content">
+        <table border="1">
+            <tr>
+                <th>Key</th>
+                <th>Value</th>
+            </tr>
+            <tr>
+                <td>Interview ID (index in results)</td>
+                <td>{{ index }}</td>
+            </tr>
+            <tr>
+                <td>Question name (question_name)</td>
+                <td>{{ question }}</td>
+            </tr>
+ 
+            <tr>
+                <td>Question type (question_type)</td>
+                <td>{{ exception_message.question_type }}</td>
+            </tr>
+ 
+            <tr>
+                <td>Human-readable question</td>
+                <td>{{ interview.survey._get_question_by_name(question).html(
+                    scenario = interview.scenario, 
+                    agent = interview.agent,
+                    answers = exception_message.answers) 
+                
+                }}</td>
+            </tr>
+            <tr>
+                <td>Scenario</td>
+                <td>{{ interview.scenario.__repr__() }}</td>
+            </tr>
+            <tr>
+                <td>Agent</td>
+                <td>{{ interview.agent.__repr__() }}</td>
+            </tr>
+            <tr>
+                <td>Model name</td>
+                <td>{{ interview.model.model }}</td>
+            </tr>
+            <tr>
+                <td>Inference service</td>
+                <td>{{ interview.model._inference_service_ }}</td>
+            </tr>
+            <tr>
+                <td>Model parameters</td>
+                <td>{{ interview.model.__repr__() }}</td>
+            </tr>
+            <tr>
+                <td>User Prompt</td>
+                <td><pre>{{ exception_message.rendered_prompts['user_prompt'] }}</pre></td>
+            </tr>
+            <tr>
+                <td>System Prompt</td>
+                <td><pre>{{ exception_message.rendered_prompts['system_prompt'] }}</pre></td>
+            </tr>
+            <tr>
+                <td>Raw model response</td>
+                <td><pre>{{ exception_message.raw_model_response }}</pre>
+                </td>
+            </tr>
+            <tr>
+                <td>Generated token string (at {{ exception_message.key_sequence }}) in raw response</td>
+                <td><pre>{{ exception_message.generated_token_string }}</pre>
+                </td>
+            </tr>
+            <tr>
+            <td>Code to (likely) reproduce the error</td>
+            <td>
+                <textarea id="codeToCopy" rows="10" cols="90">{{ exception_message.code_to_reproduce }}</textarea>
+                <button onclick="copyCode()">Copy</button>
+            </td>
+            </tr>
+
+        </table>
+
+        
+            {% if exception_message.exception.__class__.__name__ == 'QuestionAnswerValidationError' %}        
+            <h3>Answer validation details</h3>
+            <table border="1">
+                <tr>
+                    <th>Field</th>
+                    <th>Value</th>
+                </tr>
+                {% for field, (explanation, open_tag, close_tag, value) in exception_message.exception.to_html_dict().items() %}
+                
+                <tr>
+                    <td>{{ field }}: ({{ explanation }})</td>
+                    <td><{{open_tag}}> {{ value | escape }} <{{close_tag}}></td>
+                </tr>
+                {% endfor %}
+            </table>
+        {% endif %}
+
+            <div class="exception-time">Time: {{ exception_message.time }}</div>          
+            <div class="exception-traceback">Traceback: 
+                <text>
+                <pre>{{ exception_message.traceback }}</pre>
+                </text>
+            </div>
+        </div>
+    </div>
+
+{% endfor %}

edsl/templates/error_reporting/interviews.html

@@ -0,0 +1,19 @@
+
+{% if interviews|length > max_interviews %}
+    <h1>Only showing the first {{ max_interviews }} interviews with errors</h1>
+{% else %}
+    <h1>Showing all interviews</h1>    
+{% endif %}
+
+{% for index, interview in interviews.items() %}
+    {% if index < max_interviews %}
+        {% if interview.exceptions != {} %}
+        <div class="interview">Interview: {{ index }} </div>
+        Model: {{ interview.model.model }}
+        <h1>Failing questions</h1>
+        {% endif %}
+        {% for question, exceptions in interview.exceptions.items() %}
+            {% include 'interview_details.html' %}
+        {% endfor %}
+    {% endif %}            
+{% endfor %}

edsl/templates/error_reporting/overview.html

@@ -0,0 +1,5 @@
+<h1>Overview</h1>
+<p>There were {{ interviews|length }} total interview(s). An 'interview' is the result of one survey, taken by one agent, with one model, with one scenario.</p>    
+The number of interviews with any exceptions was {{ num_exceptions }}.</p>
+<p>For advice on dealing with exceptions on Expected Parrot, 
+see <a href="https://docs.expectedparrot.com/en/latest/exceptions.html">here</a>.</p>

edsl/templates/error_reporting/performance_plot.html

@@ -0,0 +1,2 @@
+<h1>Performance Plot</h1>
+{{ performance_plot_html }}

edsl/templates/error_reporting/report.css

@@ -0,0 +1,74 @@
+body {
+    font-family: Arial, sans-serif;
+    line-height: 1.6;
+    background-color: #f9f9f9;
+    color: #333;
+    margin: 20px;
+}
+
+.interview {
+    font-size: 1.5em;
+    margin-bottom: 10px;
+    padding: 10px;
+    background-color: #e3f2fd;
+    border-left: 5px solid #2196f3;
+}
+
+.question {
+    font-size: 1.2em;
+    margin-bottom: 10px;
+    padding: 10px;
+    background-color: #fff9c4;
+    border-left: 5px solid #ffeb3b;
+}
+
+.exception-detail {
+    margin-bottom: 10px;
+    background-color: #ffebee;
+    border-left: 5px solid #f44336;
+}
+
+.exception-header {
+    padding: 10px;
+    cursor: pointer;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
+.exception-content {
+    padding: 10px;
+    display: none;
+}
+
+.exception-content.show {
+    display: block;
+}
+
+.question-detail {
+    border: 3px solid black;
+    padding: 10px;
+}
+
+.exception-exception {
+    font-weight: bold;
+    color: #d32f2f;
+}
+
+.exception-time,
+.exception-traceback {
+    font-style: italic;
+    color: #555;
+}
+
+.toggle-btn {
+    background: none;
+    border: none;
+    font-size: 1.2em;
+    cursor: pointer;
+    transition: transform 0.3s;
+}
+
+.toggle-btn.rotated {
+    transform: rotate(180deg);
+}