edsl 0.1.33.dev1__py3-none-any.whl → 0.1.33.dev2__py3-none-any.whl

edsl/jobs/tasks/QuestionTaskCreator.py

@@ -55,6 +55,7 @@ class QuestionTaskCreator(UserList):
 
         """
         super().__init__([])
+        # answer_question_func is the 'interview.answer_question_and_record_task" method
         self.answer_question_func = answer_question_func
         self.question = question
         self.iteration = iteration
@@ -87,10 +88,10 @@ class QuestionTaskCreator(UserList):
         """
         self.append(task)
 
-    def generate_task(self, debug: bool) -> asyncio.Task:
+    def generate_task(self) -> asyncio.Task:
         """Create a task that depends on the passed-in dependencies."""
         task = asyncio.create_task(
-            self._run_task_async(debug), name=self.question.question_name
+            self._run_task_async(), name=self.question.question_name
         )
         task.depends_on = [t.get_name() for t in self]
         return task
@@ -103,7 +104,7 @@ class QuestionTaskCreator(UserList):
         """Returns the token usage for the task.
 
         >>> qt = QuestionTaskCreator.example()
-        >>> answers = asyncio.run(qt._run_focal_task(debug=False))
+        >>> answers = asyncio.run(qt._run_focal_task())
         >>> qt.token_usage()
         {'cached_tokens': TokenUsage(from_cache=True, prompt_tokens=0, completion_tokens=0), 'new_tokens': TokenUsage(from_cache=False, prompt_tokens=0, completion_tokens=0)}
         """
@@ -111,15 +112,15 @@ class QuestionTaskCreator(UserList):
             cached_tokens=self.cached_token_usage, new_tokens=self.new_token_usage
         )
 
-    async def _run_focal_task(self, debug: bool) -> Answers:
+    async def _run_focal_task(self) -> Answers:
         """Run the focal task i.e., the question that we are interested in answering.
 
         It is only called after all the dependency tasks are completed.
 
         >>> qt = QuestionTaskCreator.example()
-        >>> answers = asyncio.run(qt._run_focal_task(debug=False))
-        >>> answers["answer"]
-        'Yo!'
+        >>> answers = asyncio.run(qt._run_focal_task())
+        >>> answers.answer
+        'This is an example answer'
         """
 
         requested_tokens = self.estimated_tokens()
@@ -132,19 +133,19 @@ class QuestionTaskCreator(UserList):
             self.waiting = True
             self.task_status = TaskStatus.WAITING_FOR_REQUEST_CAPACITY
 
-        await self.tokens_bucket.get_tokens(1)
+        await self.requests_bucket.get_tokens(1, cheat_bucket_capacity=True)
 
         self.task_status = TaskStatus.API_CALL_IN_PROGRESS
         try:
             results = await self.answer_question_func(
-                question=self.question, debug=debug, task=None  # self
+                question=self.question, task=None  # self
             )
             self.task_status = TaskStatus.SUCCESS
         except Exception as e:
             self.task_status = TaskStatus.FAILED
             raise e
 
-        if results.get("cache_used", False):
+        if results.cache_used:
             self.tokens_bucket.add_tokens(requested_tokens)
             self.requests_bucket.add_tokens(1)
             self.from_cache = True
@@ -155,17 +156,18 @@ class QuestionTaskCreator(UserList):
             self.tokens_bucket.turbo_mode_off()
             self.requests_bucket.turbo_mode_off()
 
-        _ = results.pop("cached_response", None)
+        # breakpoint()
+        # _ = results.pop("cached_response", None)
 
-        tracker = self.cached_token_usage if self.from_cache else self.new_token_usage
+        # tracker = self.cached_token_usage if self.from_cache else self.new_token_usage
 
         # TODO: This is hacky. The 'func' call should return an object that definitely has a 'usage' key.
-        usage = results.get("usage", {"prompt_tokens": 0, "completion_tokens": 0})
-        prompt_tokens = usage.get("prompt_tokens", 0)
-        completion_tokens = usage.get("completion_tokens", 0)
-        tracker.add_tokens(
-            prompt_tokens=prompt_tokens, completion_tokens=completion_tokens
-        )
+        # usage = results.get("usage", {"prompt_tokens": 0, "completion_tokens": 0})
+        # prompt_tokens = usage.get("prompt_tokens", 0)
+        # completion_tokens = usage.get("completion_tokens", 0)
+        # tracker.add_tokens(
+        #    prompt_tokens=prompt_tokens, completion_tokens=completion_tokens
+        # )
 
         return results
 
@@ -177,8 +179,13 @@ class QuestionTaskCreator(UserList):
 
         m = ModelBuckets.infinity_bucket()
 
-        async def answer_question_func(question, debug, task):
-            return {"answer": "Yo!"}
+        from collections import namedtuple
+
+        AnswerDict = namedtuple("AnswerDict", ["answer", "cache_used"])
+        answer = AnswerDict(answer="This is an example answer", cache_used=False)
+
+        async def answer_question_func(question, task):
+            return answer
 
         return cls(
             question=QuestionFreeText.example(),
@@ -188,7 +195,7 @@ class QuestionTaskCreator(UserList):
             iteration=0,
         )
 
-    async def _run_task_async(self, debug) -> None:
+    async def _run_task_async(self) -> None:
         """Run the task asynchronously, awaiting the tasks that must be completed before this one can be run.
 
         >>> qt1 = QuestionTaskCreator.example()
@@ -231,8 +238,6 @@ class QuestionTaskCreator(UserList):
                 if isinstance(result, Exception):
                     raise result
 
-            return await self._run_focal_task(debug)
-
         except asyncio.CancelledError:
             self.task_status = TaskStatus.CANCELLED
             raise
@@ -244,6 +249,8 @@ class QuestionTaskCreator(UserList):
                 f"Required tasks failed for {self.question.question_name}"
             ) from e
 
+        return await self._run_focal_task()
+
 
 if __name__ == "__main__":
     import doctest

edsl/jobs/tasks/TaskHistory.py

@@ -1,7 +1,10 @@
-from edsl.jobs.tasks.task_status_enum import TaskStatus
 from typing import List, Optional
 from io import BytesIO
+import webbrowser
+import os
 import base64
+from importlib import resources
+from edsl.jobs.tasks.task_status_enum import TaskStatus
 
 
 class TaskHistory:
@@ -30,7 +33,12 @@ class TaskHistory:
 
         from edsl.config import CONFIG
 
-        results = j.run(print_exceptions=False, skip_retry=True, cache=False)
+        results = j.run(
+            print_exceptions=False,
+            skip_retry=True,
+            cache=False,
+            raise_validation_errors=True,
+        )
 
         return cls(results.task_history.total_interviews)
 
@@ -188,58 +196,14 @@ class TaskHistory:
             plt.show()
 
     def css(self):
-        return """
-        body {
-        font-family: Arial, sans-serif;
-        line-height: 1.6;
-        background-color: #f9f9f9;
-        color: #333;
-        margin: 20px;
-        }
-
-        .interview {
-        font-size: 1.5em;
-        margin-bottom: 10px;
-        padding: 10px;
-        background-color: #e3f2fd;
-        border-left: 5px solid #2196f3;
-        }
-
-        .question {
-        font-size: 1.2em;
-        margin-bottom: 10px;
-        padding: 10px;
-        background-color: #fff9c4;
-        border-left: 5px solid #ffeb3b;
-        }
-
-        .exception-detail {
-        margin-bottom: 10px;
-        padding: 10px;
-        background-color: #ffebee;
-        border-left: 5px solid #f44336;
-        }
-
-        .question-detail {
-           border: 3px solid black; /* Adjust the thickness by changing the number */
-            padding: 10px; /* Optional: Adds some padding inside the border */
-        }
-
-        .exception-detail div {
-        margin-bottom: 5px;
-        }
+        env = resources.files("edsl").joinpath("templates/error_reporting")
+        css = env.joinpath("report.css").read_text()
+        return css
 
-        .exception-exception {
-        font-weight: bold;
-        color: #d32f2f;
-        }
-
-        .exception-time,
-        .exception-traceback {
-        font-style: italic;
-        color: #555;
-        }
-        """
+    def javascript(self):
+        env = resources.files("edsl").joinpath("templates/error_reporting")
+        js = env.joinpath("report.js").read_text()
+        return js
 
     @property
     def exceptions_by_type(self) -> dict:
@@ -248,206 +212,136 @@ class TaskHistory:
         for interview in self.total_interviews:
             for question_name, exceptions in interview.exceptions.items():
                 for exception in exceptions:
-                    exception_type = exception["exception"]
+                    exception_type = exception.exception.__class__.__name__
+                    # exception_type = exception["exception"]
+                    # breakpoint()
                     if exception_type in exceptions_by_type:
                         exceptions_by_type[exception_type] += 1
                     else:
                         exceptions_by_type[exception_type] = 1
         return exceptions_by_type
 
+    @property
+    def exceptions_by_service(self) -> dict:
+        """Return a dictionary of exceptions tallied by service."""
+        exceptions_by_service = {}
+        for interview in self.total_interviews:
+            service = interview.model._inference_service_
+            if service not in exceptions_by_service:
+                exceptions_by_service[service] = 0
+            if interview.exceptions != {}:
+                exceptions_by_service[service] += len(interview.exceptions)
+        return exceptions_by_service
+
     @property
     def exceptions_by_question_name(self) -> dict:
         """Return a dictionary of exceptions tallied by question name."""
         exceptions_by_question_name = {}
         for interview in self.total_interviews:
             for question_name, exceptions in interview.exceptions.items():
-                if question_name not in exceptions_by_question_name:
-                    exceptions_by_question_name[question_name] = 0
-                exceptions_by_question_name[question_name] += len(exceptions)
+                question_type = interview.survey.get_question(
+                    question_name
+                ).question_type
+                # breakpoint()
+                if (question_name, question_type) not in exceptions_by_question_name:
+                    exceptions_by_question_name[(question_name, question_type)] = 0
+                exceptions_by_question_name[(question_name, question_type)] += len(
+                    exceptions
+                )
 
         for question in self.total_interviews[0].survey.questions:
-            if question.question_name not in exceptions_by_question_name:
-                exceptions_by_question_name[question.question_name] = 0
-        return exceptions_by_question_name
+            if (
+                question.question_name,
+                question.question_type,
+            ) not in exceptions_by_question_name:
+                exceptions_by_question_name[
+                    (question.question_name, question.question_type)
+                ] = 0
+
+        sorted_exceptions_by_question_name = {
+            k: v
+            for k, v in sorted(
+                exceptions_by_question_name.items(),
+                key=lambda item: item[1],
+                reverse=True,
+            )
+        }
+        return sorted_exceptions_by_question_name
 
     @property
     def exceptions_by_model(self) -> dict:
         """Return a dictionary of exceptions tallied by model and question name."""
         exceptions_by_model = {}
         for interview in self.total_interviews:
-            model = interview.model
-            if model not in exceptions_by_model:
-                exceptions_by_model[model.model] = 0
+            model = interview.model.model
+            service = interview.model._inference_service_
+            if (service, model) not in exceptions_by_model:
+                exceptions_by_model[(service, model)] = 0
             if interview.exceptions != {}:
-                exceptions_by_model[model.model] += len(interview.exceptions)
-        return exceptions_by_model
-
-    def html(
-        self,
-        filename: Optional[str] = None,
-        return_link=False,
-        css=None,
-        cta="Open Report in New Tab",
-    ):
-        """Return an HTML report."""
-
-        from IPython.display import display, HTML
-        import tempfile
-        import os
-        from edsl.utilities.utilities import is_notebook
-        from jinja2 import Template
+                exceptions_by_model[(service, model)] += len(interview.exceptions)
+
+        # sort the exceptions by model
+        sorted_exceptions_by_model = {
+            k: v
+            for k, v in sorted(
+                exceptions_by_model.items(), key=lambda item: item[1], reverse=True
+            )
+        }
+        return sorted_exceptions_by_model
 
+    def generate_html_report(self, css: Optional[str]):
         performance_plot_html = self.plot(num_periods=100, get_embedded_html=True)
 
         if css is None:
             css = self.css()
 
-        models_used = set([i.model for index, i in self._interviews.items()])
+        models_used = set([i.model.model for index, i in self._interviews.items()])
 
-        template = Template(
-            """
-        <!DOCTYPE html>
-        <html lang="en">
-        <head>
-        <meta charset="UTF-8">
-        <meta name="viewport" content="width=device-width, initial-scale=1.0">
-        <title>Exception Details</title>
-        <style>
-        {{ css }}
-        </style>
-        </head>
-        <body>
-            <h1>Overview</h1>
-            <p>There were {{ interviews|length }} total interviews. The number of interviews with exceptions was {{ num_exceptions }}.</p>
-            <p>The models used were: {{ models_used }}.</p>
-            <p>For documentation on dealing with exceptions on Expected Parrot, 
-            see <a href="https://docs.expectedparrot.com/en/latest/exceptions.html">here</a>.</p>
-
-            <h2>Exceptions by Type</h2>
-            <table>
-                <thead>
-                    <tr>
-                        <th>Exception Type</th>
-                        <th>Number</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    {% for exception_type, exceptions in exceptions_by_type.items() %}
-                        <tr>
-                            <td>{{ exception_type }}</td>
-                            <td>{{ exceptions }}</td>
-                        </tr>
-                    {% endfor %}
-                </tbody>
-            </table>
-
-
-            <h2>Exceptions by Model</h2>
-            <table>
-                <thead>
-                    <tr>
-                        <th>Model</th>
-                        <th>Number</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    {% for model, exceptions in exceptions_by_model.items() %}
-                        <tr>
-                            <td>{{ model }}</td>
-                            <td>{{ exceptions }}</td>
-                        </tr>
-                    {% endfor %}
-                </tbody>
-            </table>
-
-
-            <h2>Exceptions by Question Name</h2>
-            <table>
-                <thead>
-                    <tr>
-                        <th>Question Name</th>
-                        <th>Number of Exceptions</th>
-                    </tr>
-                </thead>
-                <tbody>
-                    {% for question_name, exception_count in exceptions_by_question_name.items() %}
-                        <tr>
-                            <td>{{ question_name }}</td>
-                            <td>{{ exception_count }}</td>
-                        </tr>
-                    {% endfor %}
-                </tbody>
-            </table>
-
-
-            {% for index, interview in interviews.items() %}
-                {% if interview.exceptions != {} %}
-                   <div class="interview">Interview: {{ index }} </div>
-                    <h1>Failing questions</h1>
-                {% endif %}
-                {% for question, exceptions in interview.exceptions.items() %}
-                    <div class="question">question_name: {{ question }}</div>
-
-                    <h2>Question</h2>
-                    <div class="question-detail"> 
-                            {{ interview.survey.get_question(question).html(interview.scenario,interview.agent) }}
-                    </div>        
-
-                    <h2>Scenario</h2>                            
-                    <div class="scenario"> 
-                            {{ interview.scenario._repr_html_() }}
-                    </div>        
-
-                    <h2>Agent</h2>
-                    <div class="agent">
-                            {{ interview.agent._repr_html_() }}
-                    </div>
-
-                    <h2>Model</h2>
-                    <div class="model">
-                            {{ interview.model._repr_html_() }}
-                    </div>
-                            
-                    <h2>Exception details</h2>
-
-                    {% for exception_message in exceptions %}
-                        <div class="exception-detail">
-                            <div class="exception-exception">Exception: {{ exception_message.exception }}</div>
-                            <div class="exception-time">Time: {{ exception_message.time }}</div>
-                            <div class="exception-traceback">Traceback: <pre>{{ exception_message.traceback }} </pre></div>
-                        </div>
-                    {% endfor %}            
-                {% endfor %}            
-            {% endfor %}
-                            
-        <h1>Performance Plot</h1>
-        {{ performance_plot_html }}
-        </body>
-        </html>
-        """
-        )
+        from jinja2 import Environment, FileSystemLoader
+        from edsl.TemplateLoader import TemplateLoader
+
+        env = Environment(loader=TemplateLoader("edsl", "templates/error_reporting"))
 
-        # breakpoint()
+        # Load and render a template
+        template = env.get_template("base.html")
+        # rendered_template = template.render(your_data=your_data)
 
         # Render the template with data
         output = template.render(
             interviews=self._interviews,
             css=css,
+            javascript=self.javascript(),
             num_exceptions=len(self.exceptions),
             performance_plot_html=performance_plot_html,
             exceptions_by_type=self.exceptions_by_type,
             exceptions_by_question_name=self.exceptions_by_question_name,
             exceptions_by_model=self.exceptions_by_model,
+            exceptions_by_service=self.exceptions_by_service,
             models_used=models_used,
         )
+        return output
+
+    def html(
+        self,
+        filename: Optional[str] = None,
+        return_link=False,
+        css=None,
+        cta="Open Report in New Tab",
+        open_in_browser=True,
+    ):
+        """Return an HTML report."""
+
+        from IPython.display import display, HTML
+        import tempfile
+        import os
+        from edsl.utilities.utilities import is_notebook
+
+        output = self.generate_html_report(css)
 
         # Save the rendered output to a file
         with open("output.html", "w") as f:
             f.write(output)
 
-        if css is None:
-            css = self.css()
-
         if filename is None:
             current_directory = os.getcwd()
             filename = tempfile.NamedTemporaryFile(
@@ -456,10 +350,7 @@ class TaskHistory:
 
         with open(filename, "w") as f:
             with open(filename, "w") as f:
-                # f.write(html_header)
-                # f.write(self._repr_html_())
                 f.write(output)
-                # f.write(html_footer)
 
         if is_notebook():
             import html
@@ -472,17 +363,44 @@ class TaskHistory:
             <iframe srcdoc="{ escaped_output }" style="width: 800px; height: 600px;"></iframe>
             """
             display(HTML(iframe))
-            # display(HTML(output))
         else:
             print(f"Exception report saved to {filename}")
-            import webbrowser
-            import os
 
+        if open_in_browser:
             webbrowser.open(f"file://{os.path.abspath(filename)}")
 
         if return_link:
             return filename
 
+    def notebook(self):
+        """Create a notebook with the HTML content embedded in the first cell, then delete the cell content while keeping the output."""
+        from nbformat import v4 as nbf
+        from nbconvert.preprocessors import ExecutePreprocessor
+        import nbformat
+        import os
+
+        # Use the existing html method to generate the HTML content
+        output_html = self.generate_html_report(css=None)
+        nb = nbf.new_notebook()
+
+        # Add a code cell that renders the HTML content
+        code_cell = nbf.new_code_cell(
+            f"""
+    from IPython.display import HTML, display
+    display(HTML('''{output_html}'''))
+            """
+        )
+        nb.cells.append(code_cell)
+
+        # Execute the notebook
+        ep = ExecutePreprocessor(timeout=600, kernel_name="python3")
+        ep.preprocess(nb, {"metadata": {"path": os.getcwd()}})
+
+        # After execution, clear the cell's source code
+        nb.cells[0].source = ""
+
+        return nb
+
 
 if __name__ == "__main__":
     import doctest