edsl 0.1.33__py3-none-any.whl → 0.1.33.dev1__py3-none-any.whl

edsl/jobs/tasks/TaskHistory.py

@@ -1,10 +1,7 @@
+from edsl.jobs.tasks.task_status_enum import TaskStatus
 from typing import List, Optional
 from io import BytesIO
-import webbrowser
-import os
 import base64
-from importlib import resources
-from edsl.jobs.tasks.task_status_enum import TaskStatus
 
 
 class TaskHistory:
@@ -33,12 +30,7 @@ class TaskHistory:
 
         from edsl.config import CONFIG
 
-        results = j.run(
-            print_exceptions=False,
-            skip_retry=True,
-            cache=False,
-            raise_validation_errors=True,
-        )
+        results = j.run(print_exceptions=False, skip_retry=True, cache=False)
 
         return cls(results.task_history.total_interviews)
 
@@ -50,18 +42,6 @@ class TaskHistory:
         """
         return [i.exceptions for k, i in self._interviews.items() if i.exceptions != {}]
 
-    @property
-    def unfixed_exceptions(self):
-        """
-        >>> len(TaskHistory.example().unfixed_exceptions)
-        4
-        """
-        return [
-            i.exceptions
-            for k, i in self._interviews.items()
-            if i.exceptions.num_unfixed() > 0
-        ]
-
     @property
     def indices(self):
         return [k for k, i in self._interviews.items() if i.exceptions != {}]
@@ -90,11 +70,6 @@ class TaskHistory:
         """
         return len(self.exceptions) > 0
 
-    @property
-    def has_unfixed_exceptions(self) -> bool:
-        """Return True if there are any exceptions."""
-        return len(self.unfixed_exceptions) > 0
-
     def _repr_html_(self):
         """Return an HTML representation of the TaskHistory."""
         from edsl.utilities.utilities import data_to_html
@@ -213,14 +188,58 @@ class TaskHistory:
             plt.show()
 
     def css(self):
-        env = resources.files("edsl").joinpath("templates/error_reporting")
-        css = env.joinpath("report.css").read_text()
-        return css
+        return """
+        body {
+        font-family: Arial, sans-serif;
+        line-height: 1.6;
+        background-color: #f9f9f9;
+        color: #333;
+        margin: 20px;
+        }
+
+        .interview {
+        font-size: 1.5em;
+        margin-bottom: 10px;
+        padding: 10px;
+        background-color: #e3f2fd;
+        border-left: 5px solid #2196f3;
+        }
+
+        .question {
+        font-size: 1.2em;
+        margin-bottom: 10px;
+        padding: 10px;
+        background-color: #fff9c4;
+        border-left: 5px solid #ffeb3b;
+        }
+
+        .exception-detail {
+        margin-bottom: 10px;
+        padding: 10px;
+        background-color: #ffebee;
+        border-left: 5px solid #f44336;
+        }
+
+        .question-detail {
+           border: 3px solid black; /* Adjust the thickness by changing the number */
+            padding: 10px; /* Optional: Adds some padding inside the border */
+        }
 
-    def javascript(self):
-        env = resources.files("edsl").joinpath("templates/error_reporting")
-        js = env.joinpath("report.js").read_text()
-        return js
+        .exception-detail div {
+        margin-bottom: 5px;
+        }
+
+        .exception-exception {
+        font-weight: bold;
+        color: #d32f2f;
+        }
+
+        .exception-time,
+        .exception-traceback {
+        font-style: italic;
+        color: #555;
+        }
+        """
 
     @property
     def exceptions_by_type(self) -> dict:
@@ -229,136 +248,206 @@ class TaskHistory:
         for interview in self.total_interviews:
             for question_name, exceptions in interview.exceptions.items():
                 for exception in exceptions:
-                    exception_type = exception.exception.__class__.__name__
-                    # exception_type = exception["exception"]
-                    # breakpoint()
+                    exception_type = exception["exception"]
                     if exception_type in exceptions_by_type:
                         exceptions_by_type[exception_type] += 1
                     else:
                         exceptions_by_type[exception_type] = 1
         return exceptions_by_type
 
-    @property
-    def exceptions_by_service(self) -> dict:
-        """Return a dictionary of exceptions tallied by service."""
-        exceptions_by_service = {}
-        for interview in self.total_interviews:
-            service = interview.model._inference_service_
-            if service not in exceptions_by_service:
-                exceptions_by_service[service] = 0
-            if interview.exceptions != {}:
-                exceptions_by_service[service] += len(interview.exceptions)
-        return exceptions_by_service
-
     @property
     def exceptions_by_question_name(self) -> dict:
         """Return a dictionary of exceptions tallied by question name."""
         exceptions_by_question_name = {}
         for interview in self.total_interviews:
             for question_name, exceptions in interview.exceptions.items():
-                question_type = interview.survey.get_question(
-                    question_name
-                ).question_type
-                # breakpoint()
-                if (question_name, question_type) not in exceptions_by_question_name:
-                    exceptions_by_question_name[(question_name, question_type)] = 0
-                exceptions_by_question_name[(question_name, question_type)] += len(
-                    exceptions
-                )
+                if question_name not in exceptions_by_question_name:
+                    exceptions_by_question_name[question_name] = 0
+                exceptions_by_question_name[question_name] += len(exceptions)
 
         for question in self.total_interviews[0].survey.questions:
-            if (
-                question.question_name,
-                question.question_type,
-            ) not in exceptions_by_question_name:
-                exceptions_by_question_name[
-                    (question.question_name, question.question_type)
-                ] = 0
-
-        sorted_exceptions_by_question_name = {
-            k: v
-            for k, v in sorted(
-                exceptions_by_question_name.items(),
-                key=lambda item: item[1],
-                reverse=True,
-            )
-        }
-        return sorted_exceptions_by_question_name
+            if question.question_name not in exceptions_by_question_name:
+                exceptions_by_question_name[question.question_name] = 0
+        return exceptions_by_question_name
 
     @property
     def exceptions_by_model(self) -> dict:
         """Return a dictionary of exceptions tallied by model and question name."""
         exceptions_by_model = {}
         for interview in self.total_interviews:
-            model = interview.model.model
-            service = interview.model._inference_service_
-            if (service, model) not in exceptions_by_model:
-                exceptions_by_model[(service, model)] = 0
+            model = interview.model
+            if model not in exceptions_by_model:
+                exceptions_by_model[model.model] = 0
             if interview.exceptions != {}:
-                exceptions_by_model[(service, model)] += len(interview.exceptions)
-
-        # sort the exceptions by model
-        sorted_exceptions_by_model = {
-            k: v
-            for k, v in sorted(
-                exceptions_by_model.items(), key=lambda item: item[1], reverse=True
-            )
-        }
-        return sorted_exceptions_by_model
+                exceptions_by_model[model.model] += len(interview.exceptions)
+        return exceptions_by_model
+
+    def html(
+        self,
+        filename: Optional[str] = None,
+        return_link=False,
+        css=None,
+        cta="Open Report in New Tab",
+    ):
+        """Return an HTML report."""
+
+        from IPython.display import display, HTML
+        import tempfile
+        import os
+        from edsl.utilities.utilities import is_notebook
+        from jinja2 import Template
 
-    def generate_html_report(self, css: Optional[str]):
         performance_plot_html = self.plot(num_periods=100, get_embedded_html=True)
 
         if css is None:
             css = self.css()
 
-        models_used = set([i.model.model for index, i in self._interviews.items()])
-
-        from jinja2 import Environment, FileSystemLoader
-        from edsl.TemplateLoader import TemplateLoader
+        models_used = set([i.model for index, i in self._interviews.items()])
 
-        env = Environment(loader=TemplateLoader("edsl", "templates/error_reporting"))
+        template = Template(
+            """
+        <!DOCTYPE html>
+        <html lang="en">
+        <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>Exception Details</title>
+        <style>
+        {{ css }}
+        </style>
+        </head>
+        <body>
+            <h1>Overview</h1>
+            <p>There were {{ interviews|length }} total interviews. The number of interviews with exceptions was {{ num_exceptions }}.</p>
+            <p>The models used were: {{ models_used }}.</p>
+            <p>For documentation on dealing with exceptions on Expected Parrot, 
+            see <a href="https://docs.expectedparrot.com/en/latest/exceptions.html">here</a>.</p>
+
+            <h2>Exceptions by Type</h2>
+            <table>
+                <thead>
+                    <tr>
+                        <th>Exception Type</th>
+                        <th>Number</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for exception_type, exceptions in exceptions_by_type.items() %}
+                        <tr>
+                            <td>{{ exception_type }}</td>
+                            <td>{{ exceptions }}</td>
+                        </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+
+
+            <h2>Exceptions by Model</h2>
+            <table>
+                <thead>
+                    <tr>
+                        <th>Model</th>
+                        <th>Number</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for model, exceptions in exceptions_by_model.items() %}
+                        <tr>
+                            <td>{{ model }}</td>
+                            <td>{{ exceptions }}</td>
+                        </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+
+
+            <h2>Exceptions by Question Name</h2>
+            <table>
+                <thead>
+                    <tr>
+                        <th>Question Name</th>
+                        <th>Number of Exceptions</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for question_name, exception_count in exceptions_by_question_name.items() %}
+                        <tr>
+                            <td>{{ question_name }}</td>
+                            <td>{{ exception_count }}</td>
+                        </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+
+
+            {% for index, interview in interviews.items() %}
+                {% if interview.exceptions != {} %}
+                   <div class="interview">Interview: {{ index }} </div>
+                    <h1>Failing questions</h1>
+                {% endif %}
+                {% for question, exceptions in interview.exceptions.items() %}
+                    <div class="question">question_name: {{ question }}</div>
+
+                    <h2>Question</h2>
+                    <div class="question-detail"> 
+                            {{ interview.survey.get_question(question).html(interview.scenario,interview.agent) }}
+                    </div>        
+
+                    <h2>Scenario</h2>                            
+                    <div class="scenario"> 
+                            {{ interview.scenario._repr_html_() }}
+                    </div>        
+
+                    <h2>Agent</h2>
+                    <div class="agent">
+                            {{ interview.agent._repr_html_() }}
+                    </div>
+
+                    <h2>Model</h2>
+                    <div class="model">
+                            {{ interview.model._repr_html_() }}
+                    </div>
+                            
+                    <h2>Exception details</h2>
+
+                    {% for exception_message in exceptions %}
+                        <div class="exception-detail">
+                            <div class="exception-exception">Exception: {{ exception_message.exception }}</div>
+                            <div class="exception-time">Time: {{ exception_message.time }}</div>
+                            <div class="exception-traceback">Traceback: <pre>{{ exception_message.traceback }} </pre></div>
+                        </div>
+                    {% endfor %}            
+                {% endfor %}            
+            {% endfor %}
+                            
+        <h1>Performance Plot</h1>
+        {{ performance_plot_html }}
+        </body>
+        </html>
+        """
+        )
 
-        # Load and render a template
-        template = env.get_template("base.html")
-        # rendered_template = template.render(your_data=your_data)
+        # breakpoint()
 
         # Render the template with data
         output = template.render(
             interviews=self._interviews,
             css=css,
-            javascript=self.javascript(),
             num_exceptions=len(self.exceptions),
             performance_plot_html=performance_plot_html,
             exceptions_by_type=self.exceptions_by_type,
             exceptions_by_question_name=self.exceptions_by_question_name,
             exceptions_by_model=self.exceptions_by_model,
-            exceptions_by_service=self.exceptions_by_service,
             models_used=models_used,
         )
-        return output
-
-    def html(
-        self,
-        filename: Optional[str] = None,
-        return_link=False,
-        css=None,
-        cta="Open Report in New Tab",
-        open_in_browser=True,
-    ):
-        """Return an HTML report."""
-
-        from IPython.display import display, HTML
-        import tempfile
-        import os
-        from edsl.utilities.utilities import is_notebook
-
-        output = self.generate_html_report(css)
 
         # Save the rendered output to a file
         with open("output.html", "w") as f:
             f.write(output)
 
+        if css is None:
+            css = self.css()
+
         if filename is None:
             current_directory = os.getcwd()
             filename = tempfile.NamedTemporaryFile(
@@ -367,7 +456,10 @@ class TaskHistory:
 
         with open(filename, "w") as f:
             with open(filename, "w") as f:
+                # f.write(html_header)
+                # f.write(self._repr_html_())
                 f.write(output)
+                # f.write(html_footer)
 
         if is_notebook():
             import html
@@ -380,44 +472,17 @@ class TaskHistory:
             <iframe srcdoc="{ escaped_output }" style="width: 800px; height: 600px;"></iframe>
             """
             display(HTML(iframe))
+            # display(HTML(output))
         else:
             print(f"Exception report saved to {filename}")
+            import webbrowser
+            import os
 
-        if open_in_browser:
             webbrowser.open(f"file://{os.path.abspath(filename)}")
 
         if return_link:
             return filename
 
-    def notebook(self):
-        """Create a notebook with the HTML content embedded in the first cell, then delete the cell content while keeping the output."""
-        from nbformat import v4 as nbf
-        from nbconvert.preprocessors import ExecutePreprocessor
-        import nbformat
-        import os
-
-        # Use the existing html method to generate the HTML content
-        output_html = self.generate_html_report(css=None)
-        nb = nbf.new_notebook()
-
-        # Add a code cell that renders the HTML content
-        code_cell = nbf.new_code_cell(
-            f"""
-    from IPython.display import HTML, display
-    display(HTML('''{output_html}'''))
-            """
-        )
-        nb.cells.append(code_cell)
-
-        # Execute the notebook
-        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")
-        ep.preprocess(nb, {"metadata": {"path": os.getcwd()}})
-
-        # After execution, clear the cell's source code
-        nb.cells[0].source = ""
-
-        return nb
-
 
 if __name__ == "__main__":
     import doctest