edsl 0.1.31.dev4__py3-none-any.whl → 0.1.33__py3-none-any.whl

edsl/jobs/tasks/TaskHistory.py

@@ -1,7 +1,10 @@
-from edsl.jobs.tasks.task_status_enum import TaskStatus
 from typing import List, Optional
 from io import BytesIO
+import webbrowser
+import os
 import base64
+from importlib import resources
+from edsl.jobs.tasks.task_status_enum import TaskStatus
 
 
 class TaskHistory:
@@ -11,6 +14,8 @@ class TaskHistory:
 
         [Interview.exceptions, Interview.exceptions, Interview.exceptions, ...]
 
+        >>> _ = TaskHistory.example()
+        ...
         """
 
         self.total_interviews = interviews
@@ -18,10 +23,45 @@ class TaskHistory:
 
         self._interviews = {index: i for index, i in enumerate(self.total_interviews)}
 
+    @classmethod
+    def example(cls):
+        from edsl.jobs.interviews.Interview import Interview
+
+        from edsl.jobs.Jobs import Jobs
+
+        j = Jobs.example(throw_exception_probability=1, test_model=True)
+
+        from edsl.config import CONFIG
+
+        results = j.run(
+            print_exceptions=False,
+            skip_retry=True,
+            cache=False,
+            raise_validation_errors=True,
+        )
+
+        return cls(results.task_history.total_interviews)
+
     @property
     def exceptions(self):
+        """
+        >>> len(TaskHistory.example().exceptions)
+        4
+        """
         return [i.exceptions for k, i in self._interviews.items() if i.exceptions != {}]
 
+    @property
+    def unfixed_exceptions(self):
+        """
+        >>> len(TaskHistory.example().unfixed_exceptions)
+        4
+        """
+        return [
+            i.exceptions
+            for k, i in self._interviews.items()
+            if i.exceptions.num_unfixed() > 0
+        ]
+
     @property
     def indices(self):
         return [k for k, i in self._interviews.items() if i.exceptions != {}]
@@ -42,9 +82,19 @@ class TaskHistory:
 
     @property
     def has_exceptions(self) -> bool:
-        """Return True if there are any exceptions."""
+        """Return True if there are any exceptions.
+
+        >>> TaskHistory.example().has_exceptions
+        True
+
+        """
         return len(self.exceptions) > 0
 
+    @property
+    def has_unfixed_exceptions(self) -> bool:
+        """Return True if there are any exceptions."""
+        return len(self.unfixed_exceptions) > 0
+
     def _repr_html_(self):
         """Return an HTML representation of the TaskHistory."""
         from edsl.utilities.utilities import data_to_html
@@ -163,58 +213,130 @@ class TaskHistory:
             plt.show()
 
     def css(self):
-        return """
-        body {
-        font-family: Arial, sans-serif;
-        line-height: 1.6;
-        background-color: #f9f9f9;
-        color: #333;
-        margin: 20px;
-        }
+        env = resources.files("edsl").joinpath("templates/error_reporting")
+        css = env.joinpath("report.css").read_text()
+        return css
 
-        .interview {
-        font-size: 1.5em;
-        margin-bottom: 10px;
-        padding: 10px;
-        background-color: #e3f2fd;
-        border-left: 5px solid #2196f3;
-        }
+    def javascript(self):
+        env = resources.files("edsl").joinpath("templates/error_reporting")
+        js = env.joinpath("report.js").read_text()
+        return js
 
-        .question {
-        font-size: 1.2em;
-        margin-bottom: 10px;
-        padding: 10px;
-        background-color: #fff9c4;
-        border-left: 5px solid #ffeb3b;
-        }
+    @property
+    def exceptions_by_type(self) -> dict:
+        """Return a dictionary of exceptions by type."""
+        exceptions_by_type = {}
+        for interview in self.total_interviews:
+            for question_name, exceptions in interview.exceptions.items():
+                for exception in exceptions:
+                    exception_type = exception.exception.__class__.__name__
+                    # exception_type = exception["exception"]
+                    # breakpoint()
+                    if exception_type in exceptions_by_type:
+                        exceptions_by_type[exception_type] += 1
+                    else:
+                        exceptions_by_type[exception_type] = 1
+        return exceptions_by_type
 
-        .exception-detail {
-        margin-bottom: 10px;
-        padding: 10px;
-        background-color: #ffebee;
-        border-left: 5px solid #f44336;
-        }
+    @property
+    def exceptions_by_service(self) -> dict:
+        """Return a dictionary of exceptions tallied by service."""
+        exceptions_by_service = {}
+        for interview in self.total_interviews:
+            service = interview.model._inference_service_
+            if service not in exceptions_by_service:
+                exceptions_by_service[service] = 0
+            if interview.exceptions != {}:
+                exceptions_by_service[service] += len(interview.exceptions)
+        return exceptions_by_service
 
-        .question-detail {
-           border: 3px solid black; /* Adjust the thickness by changing the number */
-            padding: 10px; /* Optional: Adds some padding inside the border */
+    @property
+    def exceptions_by_question_name(self) -> dict:
+        """Return a dictionary of exceptions tallied by question name."""
+        exceptions_by_question_name = {}
+        for interview in self.total_interviews:
+            for question_name, exceptions in interview.exceptions.items():
+                question_type = interview.survey.get_question(
+                    question_name
+                ).question_type
+                # breakpoint()
+                if (question_name, question_type) not in exceptions_by_question_name:
+                    exceptions_by_question_name[(question_name, question_type)] = 0
+                exceptions_by_question_name[(question_name, question_type)] += len(
+                    exceptions
+                )
+
+        for question in self.total_interviews[0].survey.questions:
+            if (
+                question.question_name,
+                question.question_type,
+            ) not in exceptions_by_question_name:
+                exceptions_by_question_name[
+                    (question.question_name, question.question_type)
+                ] = 0
+
+        sorted_exceptions_by_question_name = {
+            k: v
+            for k, v in sorted(
+                exceptions_by_question_name.items(),
+                key=lambda item: item[1],
+                reverse=True,
+            )
         }
+        return sorted_exceptions_by_question_name
 
-        .exception-detail div {
-        margin-bottom: 5px;
+    @property
+    def exceptions_by_model(self) -> dict:
+        """Return a dictionary of exceptions tallied by model and question name."""
+        exceptions_by_model = {}
+        for interview in self.total_interviews:
+            model = interview.model.model
+            service = interview.model._inference_service_
+            if (service, model) not in exceptions_by_model:
+                exceptions_by_model[(service, model)] = 0
+            if interview.exceptions != {}:
+                exceptions_by_model[(service, model)] += len(interview.exceptions)
+
+        # sort the exceptions by model
+        sorted_exceptions_by_model = {
+            k: v
+            for k, v in sorted(
+                exceptions_by_model.items(), key=lambda item: item[1], reverse=True
+            )
         }
+        return sorted_exceptions_by_model
 
-        .exception-exception {
-        font-weight: bold;
-        color: #d32f2f;
-        }
+    def generate_html_report(self, css: Optional[str]):
+        performance_plot_html = self.plot(num_periods=100, get_embedded_html=True)
 
-        .exception-time,
-        .exception-traceback {
-        font-style: italic;
-        color: #555;
-        }
-        """
+        if css is None:
+            css = self.css()
+
+        models_used = set([i.model.model for index, i in self._interviews.items()])
+
+        from jinja2 import Environment, FileSystemLoader
+        from edsl.TemplateLoader import TemplateLoader
+
+        env = Environment(loader=TemplateLoader("edsl", "templates/error_reporting"))
+
+        # Load and render a template
+        template = env.get_template("base.html")
+        # rendered_template = template.render(your_data=your_data)
+
+        # Render the template with data
+        output = template.render(
+            interviews=self._interviews,
+            css=css,
+            javascript=self.javascript(),
+            num_exceptions=len(self.exceptions),
+            performance_plot_html=performance_plot_html,
+            exceptions_by_type=self.exceptions_by_type,
+            exceptions_by_question_name=self.exceptions_by_question_name,
+            exceptions_by_model=self.exceptions_by_model,
+            exceptions_by_service=self.exceptions_by_service,
+            models_used=models_used,
+        )
+        return output
 
     def html(
         self,
@@ -222,6 +344,7 @@ class TaskHistory:
         return_link=False,
         css=None,
         cta="Open Report in New Tab",
+        open_in_browser=True,
     ):
         """Return an HTML report."""
 
@@ -229,87 +352,13 @@ class TaskHistory:
         import tempfile
         import os
         from edsl.utilities.utilities import is_notebook
-        from jinja2 import Template
 
-        performance_plot_html = self.plot(num_periods=100, get_embedded_html=True)
-
-        if css is None:
-            css = self.css()
-
-        template = Template(
-            """
-        <!DOCTYPE html>
-        <html lang="en">
-        <head>
-        <meta charset="UTF-8">
-        <meta name="viewport" content="width=device-width, initial-scale=1.0">
-        <title>Exception Details</title>
-        <style>
-        {{ css }}
-        </style>
-        </head>
-        <body>
-            {% for index, interview in interviews.items() %}
-                {% if interview.exceptions != {} %}
-                   <div class="interview">Interview: {{ index }} </div>
-                    <h1>Failing questions</h1>
-                {% endif %}
-                {% for question, exceptions in interview.exceptions.items() %}
-                    <div class="question">question_name: {{ question }}</div>
-
-                    <h2>Question</h2>
-                    <div class="question-detail"> 
-                            {{ interview.survey.get_question(question).html() }}
-                    </div>        
-
-                    <h2>Scenario</h2>                            
-                    <div class="scenario"> 
-                            {{ interview.scenario._repr_html_() }}
-                    </div>        
-
-                    <h2>Agent</h2>
-                    <div class="agent">
-                            {{ interview.agent._repr_html_() }}
-                    </div>
-
-                    <h2>Model</h2>
-                    <div class="model">
-                            {{ interview.model._repr_html_() }}
-                    </div>
-                            
-                    <h2>Exception details</h2>
-
-                    {% for exception_message in exceptions %}
-                        <div class="exception-detail">
-                            <div class="exception-exception">Exception: {{ exception_message.exception }}</div>
-                            <div class="exception-time">Time: {{ exception_message.time }}</div>
-                            <div class="exception-traceback">Traceback: <pre>{{ exception_message.traceback }} </pre></div>
-                        </div>
-                    {% endfor %}            
-                {% endfor %}            
-            {% endfor %}
-                            
-        <h1>Performance Plot</h1>
-        {{ performance_plot_html }}
-        </body>
-        </html>
-        """
-        )
-
-        # Render the template with data
-        output = template.render(
-            interviews=self._interviews,
-            css=css,
-            performance_plot_html=performance_plot_html,
-        )
+        output = self.generate_html_report(css)
 
         # Save the rendered output to a file
         with open("output.html", "w") as f:
             f.write(output)
 
-        if css is None:
-            css = self.css()
-
         if filename is None:
             current_directory = os.getcwd()
             filename = tempfile.NamedTemporaryFile(
@@ -318,10 +367,7 @@ class TaskHistory:
 
         with open(filename, "w") as f:
             with open(filename, "w") as f:
-                # f.write(html_header)
-                # f.write(self._repr_html_())
                 f.write(output)
-                # f.write(html_footer)
 
         if is_notebook():
             import html
@@ -334,13 +380,46 @@ class TaskHistory:
             <iframe srcdoc="{ escaped_output }" style="width: 800px; height: 600px;"></iframe>
             """
             display(HTML(iframe))
-            # display(HTML(output))
         else:
             print(f"Exception report saved to {filename}")
-            import webbrowser
-            import os
 
+        if open_in_browser:
             webbrowser.open(f"file://{os.path.abspath(filename)}")
 
         if return_link:
             return filename
+
+    def notebook(self):
+        """Create a notebook with the HTML content embedded in the first cell, then delete the cell content while keeping the output."""
+        from nbformat import v4 as nbf
+        from nbconvert.preprocessors import ExecutePreprocessor
+        import nbformat
+        import os
+
+        # Use the existing html method to generate the HTML content
+        output_html = self.generate_html_report(css=None)
+        nb = nbf.new_notebook()
+
+        # Add a code cell that renders the HTML content
+        code_cell = nbf.new_code_cell(
+            f"""
+    from IPython.display import HTML, display
+    display(HTML('''{output_html}'''))
+            """
+        )
+        nb.cells.append(code_cell)
+
+        # Execute the notebook
+        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")
+        ep.preprocess(nb, {"metadata": {"path": os.getcwd()}})
+
+        # After execution, clear the cell's source code
+        nb.cells[0].source = ""
+
+        return nb
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)