edsl 0.1.32__py3-none-any.whl → 0.1.33__py3-none-any.whl

edsl/surveys/SurveyExportMixin.py

@@ -2,6 +2,27 @@
 
 from typing import Union, Optional
 
+import subprocess
+import platform
+import os
+import tempfile
+
+
+def open_docx(file_path):
+    """
+    Open a docx file using the default application in a cross-platform manner.
+
+    :param file_path: str, path to the docx file
+    """
+    file_path = os.path.abspath(file_path)
+
+    if platform.system() == "Darwin":  # macOS
+        subprocess.call(("open", file_path))
+    elif platform.system() == "Windows":  # Windows
+        os.startfile(file_path)
+    else:  # linux variants
+        subprocess.call(("xdg-open", file_path))
+
 
 class SurveyExportMixin:
     """A mixin class for exporting surveys to different formats."""
@@ -25,7 +46,12 @@ class SurveyExportMixin:
         )
         return q.run().select("description").first()
 
-    def docx(self, filename=None) -> Union["Document", None]:
+    def docx(
+        self,
+        return_document_object: bool = False,
+        filename: Optional[str] = None,
+        open_file: bool = False,
+    ) -> Union["Document", None]:
         """Generate a docx document for the survey."""
         from docx import Document
 
@@ -45,19 +71,55 @@ class SurveyExportMixin:
                 if hasattr(question, "question_options"):
                     for option in getattr(question, "question_options", []):
                         doc.add_paragraph(str(option), style="ListBullet")
-        if filename:
-            doc.save(filename)
-            print("The survey has been saved to", filename)
-            return
-        return doc
 
-    def to_scenario_list(self) -> "ScenarioList":
+        if return_document_object and filename is None:
+            return doc
+
+        if filename is None:
+            with tempfile.NamedTemporaryFile(
+                "w", delete=False, suffix=".docx", dir=os.getcwd()
+            ) as f:
+                filename = f.name
+
+        doc.save(filename)
+        print("The survey has been saved to", filename)
+        if open_file:
+            open_docx(filename)
+        return
+
+    def show(self):
+        self.to_scenario_list(questions_only=False, rename=True).print(format="rich")
+
+    def to_scenario_list(
+        self, questions_only: bool = True, rename=False
+    ) -> "ScenarioList":
         from edsl import ScenarioList, Scenario
 
+        # from edsl.questions import QuestionBase
+
+        if questions_only:
+            to_iterate_over = self._questions
+        else:
+            to_iterate_over = self.recombined_questions_and_instructions()
+
+        if rename:
+            renaming_dict = {
+                "name": "identifier",
+                "question_name": "identifier",
+                "question_text": "text",
+            }
+        else:
+            renaming_dict = {}
+
         all_keys = set([])
         scenarios = ScenarioList()
-        for q in self._questions:
-            d = q.to_dict()
+        for item in to_iterate_over:
+            d = item.to_dict()
+            if item.__class__.__name__ == "Instruction":
+                d["question_type"] = "NA / instruction"
+            for key in renaming_dict:
+                if key in d:
+                    d[renaming_dict[key]] = d.pop(key)
             all_keys.update(d.keys())
             scenarios.append(Scenario(d))
 

edsl/surveys/SurveyFlowVisualizationMixin.py

@@ -1,5 +1,6 @@
 """A mixin for visualizing the flow of a survey."""
 
+from typing import Optional
 from edsl.surveys.base import RulePriority, EndOfSurvey
 import tempfile
 
@@ -7,7 +8,7 @@ import tempfile
 class SurveyFlowVisualizationMixin:
     """A mixin for visualizing the flow of a survey."""
 
-    def show_flow(self, filename: str = None):
+    def show_flow(self, filename: Optional[str] = None):
         """Create an image showing the flow of users through the survey."""
         # Create a graph object
         import pydot

edsl/surveys/SurveyQualtricsImport.py

@@ -9,7 +9,7 @@ qualtrics_codes = {
     "TE": "free_text",
     "MC": "multiple_choice",
     "Matrix": "matrix",
-    "DB": "free_text",  # not quite right, but for now
+    "DB": "instruction",  # not quite right, but for now
     "Timing": "free_text",  # not quite right, but for now
 }
 # TE (Text Entry): Allows respondents to input a text response.
@@ -84,6 +84,11 @@ class QualtricsQuestion:
         return None
 
     def to_edsl(self):
+        if self.question_type == "instruction":
+            from edsl import Instruction
+
+            return [Instruction(text=self.question_text, name=self.question_name)]
+
         if self.question_type == "free_text":
             try:
                 q = Question(
@@ -187,11 +192,14 @@ class SurveyQualtricsImport:
             questions.extend(qualtrics_questions.to_edsl())
         return Survey(questions)
 
-    def extract_questions_from_json(self):
+    @property
+    def survey_data(self):
         with open(self.qsf_file_name, "r") as f:
             survey_data = json.load(f)
+        return survey_data
 
-        questions = survey_data["SurveyElements"]
+    def extract_questions_from_json(self):
+        questions = self.survey_data["SurveyElements"]
 
         extracted_questions = []
 
@@ -201,11 +209,74 @@ class SurveyQualtricsImport:
 
         return extracted_questions
 
+    def extract_blocks_from_json(self):
+        blocks = []
+
+        for element in self.survey_data["SurveyElements"]:
+            if element["Element"] == "BL":
+                for block_payload in element["Payload"]:
+                    block_elements = [
+                        BlockElement(be["Type"], be["QuestionID"])
+                        for be in block_payload["BlockElements"]
+                    ]
+                    options_data = block_payload.get("Options", {})
+                    options = BlockOptions(
+                        options_data.get("BlockLocking", "false"),
+                        options_data.get("RandomizeQuestions", "false"),
+                        options_data.get("BlockVisibility", "Collapsed"),
+                    )
+
+                    block = SurveyBlock(
+                        block_payload["Type"],
+                        block_payload["Description"],
+                        block_payload["ID"],
+                        block_elements,
+                        options,
+                    )
+                    blocks.append(block)
+
+        return blocks
+
+
+class SurveyBlock:
+    def __init__(self, block_type, description, block_id, block_elements, options):
+        self.block_type = block_type
+        self.description = description
+        self.block_id = block_id
+        self.block_elements = block_elements
+        self.options = options
+
+    def __repr__(self):
+        return f"SurveyBlock(type={self.block_type}, description={self.description}, id={self.block_id})"
+
+
+class BlockElement:
+    def __init__(self, element_type, question_id):
+        self.element_type = element_type
+        self.question_id = question_id
+
+    def __repr__(self):
+        return f"BlockElement(type={self.element_type}, question_id={self.question_id})"
+
+
+class BlockOptions:
+    def __init__(self, block_locking, randomize_questions, block_visibility):
+        self.block_locking = block_locking
+        self.randomize_questions = randomize_questions
+        self.block_visibility = block_visibility
+
+    def __repr__(self):
+        return (
+            f"BlockOptions(block_locking={self.block_locking}, "
+            f"randomize_questions={self.randomize_questions}, "
+            f"block_visibility={self.block_visibility})"
+        )
+
 
 if __name__ == "__main__":
     survey_creator = SurveyQualtricsImport("example.qsf")
     # print(survey_creator.question_data)
-    survey = survey_creator.create_survey()
+    # survey = survey_creator.create_survey()
     # info = survey.push()
     # print(info)
     # questions = survey.extract_questions_from_json()

edsl/surveys/instructions/ChangeInstruction.py

@@ -0,0 +1,47 @@
+from typing import List, Optional
+
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
+
+
+class ChangeInstruction:
+    def __init__(
+        self,
+        keep: Optional[List[str]] = None,
+        drop: Optional[List[str]] = None,
+    ):
+        if keep is None and drop is None:
+            raise ValueError("Keep and drop cannot both be None")
+
+        self.keep = keep or []
+        self.drop = drop or []
+
+    def include_instruction(self, instruction_name) -> bool:
+        return (instruction_name in self.keep) or (not instruction_name in self.drop)
+
+    def add_name(self, index) -> None:
+        self.name = "change_instruction_{}".format(index)
+
+    def __str__(self):
+        return self.text
+
+    def _to_dict(self):
+        return {
+            "keep": self.keep,
+            "drop": self.drop,
+            "edsl_class_name": "ChangeInstruction",
+        }
+
+    @add_edsl_version
+    def to_dict(self):
+        return self._to_dict()
+
+    def __hash__(self) -> int:
+        """Return a hash of the question."""
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self._to_dict())
+
+    @classmethod
+    @remove_edsl_version
+    def from_dict(cls, data):
+        return cls(data["keep"], data["drop"])

edsl/surveys/instructions/Instruction.py

@@ -0,0 +1,34 @@
+from typing import Union, Optional, List, Generator, Dict
+from edsl.questions import QuestionBase
+
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
+
+
+class Instruction:
+    def __init__(self, name, text):
+        self.name = name
+        self.text = text
+
+    def __str__(self):
+        return self.text
+
+    def __repr__(self):
+        return """Instruction(name="{}", text="{}")""".format(self.name, self.text)
+
+    def _to_dict(self):
+        return {"name": self.name, "text": self.text, "edsl_class_name": "Instruction"}
+
+    @add_edsl_version
+    def to_dict(self):
+        return self._to_dict()
+
+    def __hash__(self) -> int:
+        """Return a hash of the question."""
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self._to_dict())
+
+    @classmethod
+    @remove_edsl_version
+    def from_dict(cls, data):
+        return cls(data["name"], data["text"])

edsl/surveys/instructions/InstructionCollection.py

@@ -0,0 +1,77 @@
+from edsl.surveys.instructions.Instruction import Instruction
+from edsl.surveys.instructions.ChangeInstruction import ChangeInstruction
+from edsl.questions import QuestionBase
+from typing import Union, Optional, List, Generator, Dict
+
+
+from collections import UserDict
+
+
+class InstructionCollection(UserDict):
+    def __init__(
+        self,
+        instruction_names_to_instruction: Dict[str, Instruction],
+        questions: List[QuestionBase],
+    ):
+        self.instruction_names_to_instruction = instruction_names_to_instruction
+        self.questions = questions
+        data = {}
+        for question in self.questions:
+            # just add both the question and the question name
+            data[question.name] = list(self.relevant_instructions(question))
+            # data[question] = list(self.relevant_instructions(question))
+        super().__init__(data)
+
+    def __getitem__(self, key):
+        # in case the person uses question instead of the name
+        if isinstance(key, QuestionBase):
+            key = key.name
+        return self.data[key]
+
+    @property
+    def question_names(self):
+        return [q.name for q in self.questions]
+
+    def question_index(self, question_name):
+        return self.question_names.index(question_name)
+
+    def _entries_before(
+        self, question_name
+    ) -> tuple[List[Instruction], List[ChangeInstruction]]:
+        if question_name not in self.question_names:
+            raise ValueError(
+                f"Question name not found in the list of questions: got '{question_name}'; list is {self.question_names}"
+            )
+        instructions, changes = [], []
+
+        index = self.question_index(question_name)
+        for instruction in self.instruction_names_to_instruction.values():
+            if instruction.pseudo_index < index:
+                if isinstance(instruction, Instruction):
+                    instructions.append(instruction)
+                elif isinstance(instruction, ChangeInstruction):
+                    changes.append(instruction)
+        return instructions, changes
+
+    def relevant_instructions(
+        self, question: Union[str, QuestionBase]
+    ) -> Generator[Instruction, None, None]:
+        ## Find all the questions that are after a given instruction
+        if isinstance(question, str):
+            question_name = question
+        elif isinstance(question, QuestionBase):
+            question_name = question.name
+
+        instructions_before, changes_before = self._entries_before(question_name)
+        keep_list = []
+        drop_list = []
+        for change in changes_before:
+            keep_list.extend(change.keep)
+            drop_list.extend(change.drop)
+
+        for instruction in instructions_before:
+            if instruction.name in keep_list or instruction.name not in drop_list:
+                yield instruction
+
+    def __len__(self):
+        return len(self.instruction_names_to_instruction)

edsl/templates/error_reporting/base.html

@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Exception Details</title>
+    <style>
+    {{ css }}
+    </style>
+
+    <script>
+    {{ javascript }}
+    </script>
+
+</head>
+<body>
+    {% include 'overview.html' %}
+    {% include 'exceptions_by_type.html' %}
+    {% include 'exceptions_by_model.html' %}
+    {% include 'exceptions_by_question_name.html' %}
+    {% include 'interviews.html' %}
+    {% include 'performance_plot.html' %}
+</body>
+</html>

edsl/templates/error_reporting/exceptions_by_model.html

@@ -0,0 +1,35 @@
+<h2>Exceptions by Model</h2>
+<table border="1">
+    <thead>
+        <tr>
+            <th>Model</th>
+            <th>Number</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for model, exceptions in exceptions_by_model.items() %}
+            <tr>
+                <td>{{ model }}</td>
+                <td>{{ exceptions }}</td>
+            </tr>
+        {% endfor %}
+    </tbody>
+</table>
+
+<h2>Exceptions by Service</h2>
+<table border="1">
+    <thead>
+        <tr>
+            <th>Service</th>
+            <th>Number</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for service, exceptions in exceptions_by_service.items() %}
+            <tr>
+                <td>{{ service }}</td>
+                <td>{{ exceptions }}</td>
+            </tr>
+        {% endfor %}
+    </tbody>
+</table>

edsl/templates/error_reporting/exceptions_by_question_name.html

@@ -0,0 +1,17 @@
+<h2>Exceptions by Question Name</h2>
+<table border="1">
+    <thead>
+        <tr>
+            <th>Question Name</th>
+            <th>Number of Exceptions</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for question_name, exception_count in exceptions_by_question_name.items() %}
+            <tr>
+                <td>{{ question_name }}</td>
+                <td>{{ exception_count }}</td>
+            </tr>
+        {% endfor %}
+    </tbody>
+</table>

edsl/templates/error_reporting/exceptions_by_type.html

@@ -0,0 +1,17 @@
+<h2>Exceptions by Type</h2>
+<table border="1">
+    <thead>
+        <tr>
+            <th>Exception Type</th>
+            <th>Number</th>
+        </tr>
+    </thead>
+    <tbody>
+        {% for exception_type, exceptions in exceptions_by_type.items() %}
+            <tr>
+                <td>{{ exception_type }}</td>
+                <td>{{ exceptions }}</td>
+            </tr>
+        {% endfor %}
+    </tbody>
+</table>

edsl/templates/error_reporting/interview_details.html

@@ -0,0 +1,116 @@
+<div class="question">question_name: {{ question }}</div>
+
+
+<h2>Exception details</h2>
+
+{% for exception_message in exceptions %}
+<div class="exception-detail">
+        <div class="exception-header">
+        <span class="exception-exception">Exception: {{ exception_message.name }}</span>
+        <button class="toggle-btn">▼</button>
+        </div>
+        <div class="exception-content">
+        <table border="1">
+            <tr>
+                <th>Key</th>
+                <th>Value</th>
+            </tr>
+            <tr>
+                <td>Interview ID (index in results)</td>
+                <td>{{ index }}</td>
+            </tr>
+            <tr>
+                <td>Question name (question_name)</td>
+                <td>{{ question }}</td>
+            </tr>
+ 
+            <tr>
+                <td>Question type (question_type)</td>
+                <td>{{ exception_message.question_type }}</td>
+            </tr>
+ 
+            <tr>
+                <td>Human-readable question</td>
+                <td>{{ interview.survey.get_question(question).html(
+                    scenario = interview.scenario, 
+                    agent = interview.agent,
+                    answers = exception_message.answers) 
+                
+                }}</td>
+            </tr>
+            <tr>
+                <td>Scenario</td>
+                <td>{{ interview.scenario._repr_html_() }}</td>
+            </tr>
+            <tr>
+                <td>Agent</td>
+                <td>{{ interview.agent._repr_html_() }}</td>
+            </tr>
+            <tr>
+                <td>Model name</td>
+                <td>{{ interview.model.model }}</td>
+            </tr>
+            <tr>
+                <td>Inference service</td>
+                <td>{{ interview.model._inference_service_ }}</td>
+            </tr>
+            <tr>
+                <td>Model parameters</td>
+                <td>{{ interview.model._repr_html_() }}</td>
+            </tr>
+            <tr>
+                <td>User Prompt</td>
+                <td><pre>{{ exception_message.rendered_prompts['user_prompt'] }}</pre></td>
+            </tr>
+            <tr>
+                <td>System Prompt</td>
+                <td><pre>{{ exception_message.rendered_prompts['system_prompt'] }}</pre></td>
+            </tr>
+            <tr>
+                <td>Raw model response</td>
+                <td><pre>{{ exception_message.raw_model_response }}</pre>
+                </td>
+            </tr>
+            <tr>
+                <td>Generated token string (at {{ exception_message.key_sequence }}) in raw response</td>
+                <td><pre>{{ exception_message.generated_token_string }}</pre>
+                </td>
+            </tr>
+            <tr>
+            <td>Code to (likely) reproduce the error</td>
+            <td>
+                <textarea id="codeToCopy" rows="10" cols="90">{{ exception_message.code_to_reproduce }}</textarea>
+                <button onclick="copyCode()">Copy</button>
+            </td>
+            </tr>
+
+        </table>
+
+        
+            {% if exception_message.exception.__class__.__name__ == 'QuestionAnswerValidationError' %}        
+            <h3>Answer validation details</h3>
+            <table border="1">
+                <tr>
+                    <th>Field</th>
+                    <th>Value</th>
+                </tr>
+                {% for field, (explanation, open_tag, close_tag, value) in exception_message.exception.to_html_dict().items() %}
+                
+                <tr>
+                    <td>{{ field }}: ({{ explanation }})</td>
+                    <td><{{open_tag}}> {{ value | escape }} <{{close_tag}}></td>
+                </tr>
+                {% endfor %}
+            </table>
+        {% endif %}
+
+            <div class="exception-time">Time: {{ exception_message.time }}</div>          
+            <div class="exception-traceback">Traceback: 
+                <text>
+                <pre>{{ exception_message.traceback }}</pre>
+                </text>
+            </div>
+        </div>
+    </div>
+
+{% endfor %}

edsl/templates/error_reporting/interviews.html

@@ -0,0 +1,10 @@
+{% for index, interview in interviews.items() %}
+    {% if interview.exceptions != {} %}
+       <div class="interview">Interview: {{ index }} </div>
+       Model: {{ interview.model.model }}
+    <h1>Failing questions</h1>
+    {% endif %}
+    {% for question, exceptions in interview.exceptions.items() %}
+        {% include 'interview_details.html' %}
+    {% endfor %}            
+{% endfor %}

edsl/templates/error_reporting/overview.html

@@ -0,0 +1,5 @@
+<h1>Overview</h1>
+<p>There were {{ interviews|length }} total interview(s). An 'interview' is the result of one survey, taken by one agent, with one model, with one scenario.</p>    
+The number of interviews with any exceptions was {{ num_exceptions }}.</p>
+<p>For advice on dealing with exceptions on Expected Parrot, 
+see <a href="https://docs.expectedparrot.com/en/latest/exceptions.html">here</a>.</p>

edsl/templates/error_reporting/performance_plot.html

@@ -0,0 +1,2 @@
+<h1>Performance Plot</h1>
+{{ performance_plot_html }}