edsl 0.1.31.dev3__py3-none-any.whl → 0.1.32__py3-none-any.whl

edsl/jobs/interviews/Interview.py

@@ -14,8 +14,8 @@ from edsl.jobs.tasks.TaskCreators import TaskCreators
 from edsl.jobs.interviews.InterviewStatusLog import InterviewStatusLog
 from edsl.jobs.interviews.interview_exception_tracking import (
     InterviewExceptionCollection,
-    InterviewExceptionEntry,
 )
+from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.interviews.InterviewTaskBuildingMixin import InterviewTaskBuildingMixin
 from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
@@ -44,6 +44,7 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         iteration: int = 0,
         cache: Optional["Cache"] = None,
         sidecar_model: Optional["LanguageModel"] = None,
+        skip_retry=False,
     ):
         """Initialize the Interview instance.
 
@@ -87,6 +88,7 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         self.task_creators = TaskCreators()  # tracks the task creators
         self.exceptions = InterviewExceptionCollection()
         self._task_status_log_dict = InterviewStatusLog()
+        self.skip_retry = skip_retry
 
         # dictionary mapping question names to their index in the survey.
         self.to_index = {
@@ -94,6 +96,30 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
             for index, question_name in enumerate(self.survey.question_names)
         }
 
+    def _to_dict(self, include_exceptions=False) -> dict[str, Any]:
+        """Return a dictionary representation of the Interview instance.
+        This is just for hashing purposes.
+
+        >>> i = Interview.example()
+        >>> hash(i)
+        1646262796627658719
+        """
+        d = {
+            "agent": self.agent._to_dict(),
+            "survey": self.survey._to_dict(),
+            "scenario": self.scenario._to_dict(),
+            "model": self.model._to_dict(),
+            "iteration": self.iteration,
+            "exceptions": {},
+        }
+        if include_exceptions:
+            d["exceptions"] = self.exceptions.to_dict()
+
+    def __hash__(self) -> int:
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self._to_dict())
+
     async def async_conduct_interview(
         self,
         *,
@@ -134,8 +160,7 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         <BLANKLINE>
 
         >>> i.exceptions
-        {'q0': [{'exception': "Exception('This is a test error')", 'time': ..., 'traceback': ...
-
+        {'q0': ...
         >>> i = Interview.example()
         >>> result, _ = asyncio.run(i.async_conduct_interview(stop_on_exception = True))
         Traceback (most recent call last):
@@ -204,13 +229,9 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         {}
         >>> i._record_exception(i.tasks[0], Exception("An exception occurred."))
         >>> i.exceptions
-        {'q0': [{'exception': "Exception('An exception occurred.')", 'time': ..., 'traceback': 'NoneType: None\\n'}]}
+        {'q0': ...
         """
-        exception_entry = InterviewExceptionEntry(
-            exception=repr(exception),
-            time=time.time(),
-            traceback=traceback.format_exc(),
-        )
+        exception_entry = InterviewExceptionEntry(exception)
         self.exceptions.add(task.get_name(), exception_entry)
 
     @property
@@ -251,6 +272,7 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
             model=self.model,
             iteration=iteration,
             cache=cache,
+            skip_retry=self.skip_retry,
         )
 
     @classmethod

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -0,0 +1,101 @@
+import traceback
+import datetime
+import time
+from collections import UserDict
+
+# traceback=traceback.format_exc(),
+# traceback = frame_summary_to_dict(traceback.extract_tb(e.__traceback__))
+# traceback = [frame_summary_to_dict(f) for f in traceback.extract_tb(e.__traceback__)]
+
+
+class InterviewExceptionEntry:
+    """Class to record an exception that occurred during the interview.
+
+    >>> entry = InterviewExceptionEntry.example()
+    >>> entry.to_dict()['exception']
+    "ValueError('An error occurred.')"
+    """
+
+    def __init__(self, exception: Exception, traceback_format="html"):
+        self.time = datetime.datetime.now().isoformat()
+        self.exception = exception
+        self.traceback_format = traceback_format
+
+    def __getitem__(self, key):
+        # Support dict-like access obj['a']
+        return str(getattr(self, key))
+
+    @classmethod
+    def example(cls):
+        try:
+            raise ValueError("An error occurred.")
+        except Exception as e:
+            entry = InterviewExceptionEntry(e)
+        return entry
+
+    @property
+    def traceback(self):
+        """Return the exception as HTML."""
+        if self.traceback_format == "html":
+            return self.html_traceback
+        else:
+            return self.text_traceback
+
+    @property
+    def text_traceback(self):
+        """
+        >>> entry = InterviewExceptionEntry.example()
+        >>> entry.text_traceback
+        'Traceback (most recent call last):...'
+        """
+        e = self.exception
+        tb_str = "".join(traceback.format_exception(type(e), e, e.__traceback__))
+        return tb_str
+
+    @property
+    def html_traceback(self):
+        from rich.console import Console
+        from rich.table import Table
+        from rich.traceback import Traceback
+
+        from io import StringIO
+
+        html_output = StringIO()
+
+        console = Console(file=html_output, record=True)
+
+        tb = Traceback.from_exception(
+            type(self.exception),
+            self.exception,
+            self.exception.__traceback__,
+            show_locals=True,
+        )
+        console.print(tb)
+        return html_output.getvalue()
+
+    def to_dict(self) -> dict:
+        """Return the exception as a dictionary.
+
+        >>> entry = InterviewExceptionEntry.example()
+        >>> entry.to_dict()['exception']
+        "ValueError('An error occurred.')"
+
+        """
+        return {
+            "exception": repr(self.exception),
+            "time": self.time,
+            "traceback": self.traceback,
+        }
+
+    def push(self):
+        from edsl import Coop
+
+        coop = Coop()
+        results = coop.error_create(self.to_dict())
+        return results
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/jobs/interviews/InterviewTaskBuildingMixin.py

@@ -12,16 +12,34 @@ from edsl.exceptions import InterviewTimeoutError
 # from edsl.questions.QuestionBase import QuestionBase
 from edsl.surveys.base import EndOfSurvey
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
-from edsl.jobs.interviews.interview_exception_tracking import InterviewExceptionEntry
+from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.tasks.task_status_enum import TaskStatus
 from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
 
 # from edsl.agents.InvigilatorBase import InvigilatorBase
 
+from rich.console import Console
+from rich.traceback import Traceback
+
 TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
 
 
+def frame_summary_to_dict(frame):
+    """
+    Convert a FrameSummary object to a dictionary.
+
+    :param frame: A traceback FrameSummary object
+    :return: A dictionary containing the frame's details
+    """
+    return {
+        "filename": frame.filename,
+        "lineno": frame.lineno,
+        "name": frame.name,
+        "line": frame.line,
+    }
+
+
 class InterviewTaskBuildingMixin:
     def _build_invigilators(
         self, debug: bool
@@ -148,7 +166,6 @@ class InterviewTaskBuildingMixin:
                 raise ValueError(f"Prompt is of type {type(prompt)}")
         return len(combined_text) / 4.0
 
-    @retry_strategy
     async def _answer_question_and_record_task(
         self,
         *,
@@ -163,22 +180,29 @@ class InterviewTaskBuildingMixin:
         """
         from edsl.data_transfer_models import AgentResponseDict
 
-        try:
-            invigilator = self._get_invigilator(question, debug=debug)
+        async def _inner():
+            try:
+                invigilator = self._get_invigilator(question, debug=debug)
 
-            if self._skip_this_question(question):
-                return invigilator.get_failed_task_result()
+                if self._skip_this_question(question):
+                    return invigilator.get_failed_task_result()
 
-            response: AgentResponseDict = await self._attempt_to_answer_question(
-                invigilator, task
-            )
+                response: AgentResponseDict = await self._attempt_to_answer_question(
+                    invigilator, task
+                )
 
-            self._add_answer(response=response, question=question)
+                self._add_answer(response=response, question=question)
 
-            self._cancel_skipped_questions(question)
-            return AgentResponseDict(**response)
-        except Exception as e:
-            raise e
+                self._cancel_skipped_questions(question)
+                return AgentResponseDict(**response)
+            except Exception as e:
+                raise e
+
+        skip_rety = getattr(self, "skip_retry", False)
+        if not skip_rety:
+            _inner = retry_strategy(_inner)
+
+        return await _inner()
 
     def _add_answer(
         self, response: "AgentResponseDict", question: "QuestionBase"
@@ -203,38 +227,30 @@ class InterviewTaskBuildingMixin:
         )
         return skip
 
+    def _handle_exception(self, e, question_name: str, task=None):
+        exception_entry = InterviewExceptionEntry(e)
+        if task:
+            task.task_status = TaskStatus.FAILED
+        self.exceptions.add(question_name, exception_entry)
+
     async def _attempt_to_answer_question(
-        self, invigilator: InvigilatorBase, task: asyncio.Task
-    ) -> AgentResponseDict:
+        self, invigilator: "InvigilatorBase", task: asyncio.Task
+    ) -> "AgentResponseDict":
         """Attempt to answer the question, and handle exceptions.
 
         :param invigilator: the invigilator that will answer the question.
         :param task: the task that is being run.
+
         """
         try:
             return await asyncio.wait_for(
                 invigilator.async_answer_question(), timeout=TIMEOUT
             )
         except asyncio.TimeoutError as e:
-            exception_entry = InterviewExceptionEntry(
-                exception=repr(e),
-                time=time.time(),
-                traceback=traceback.format_exc(),
-            )
-            if task:
-                task.task_status = TaskStatus.FAILED
-            self.exceptions.add(invigilator.question.question_name, exception_entry)
-
+            self._handle_exception(e, invigilator.question.question_name, task)
             raise InterviewTimeoutError(f"Task timed out after {TIMEOUT} seconds.")
         except Exception as e:
-            exception_entry = InterviewExceptionEntry(
-                exception=repr(e),
-                time=time.time(),
-                traceback=traceback.format_exc(),
-            )
-            if task:
-                task.task_status = TaskStatus.FAILED
-            self.exceptions.add(invigilator.question.question_name, exception_entry)
+            self._handle_exception(e, invigilator.question.question_name, task)
             raise e
 
     def _cancel_skipped_questions(self, current_question: QuestionBase) -> None:

edsl/jobs/interviews/interview_exception_tracking.py

@@ -1,18 +1,70 @@
-from rich.console import Console
-from rich.table import Table
+import traceback
+import datetime
+import time
 from collections import UserDict
 
+from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 
-class InterviewExceptionEntry(UserDict):
-    """Class to record an exception that occurred during the interview."""
+#                 #traceback=traceback.format_exc(),
+#                 #traceback = frame_summary_to_dict(traceback.extract_tb(e.__traceback__))
+#                 #traceback = [frame_summary_to_dict(f) for f in traceback.extract_tb(e.__traceback__)]
 
-    def __init__(self, exception, time, traceback):
-        data = {"exception": exception, "time": time, "traceback": traceback}
-        super().__init__(data)
+# class InterviewExceptionEntry:
+#     """Class to record an exception that occurred during the interview.
 
-    def to_dict(self) -> dict:
-        """Return the exception as a dictionary."""
-        return self.data
+#     >>> entry = InterviewExceptionEntry.example()
+#     >>> entry.to_dict()['exception']
+#     "ValueError('An error occurred.')"
+#     """
+
+#     def __init__(self, exception: Exception):
+#         self.time = datetime.datetime.now().isoformat()
+#         self.exception = exception
+
+#     def __getitem__(self, key):
+#         # Support dict-like access obj['a']
+#         return str(getattr(self, key))
+
+#     @classmethod
+#     def example(cls):
+#         try:
+#             raise ValueError("An error occurred.")
+#         except Exception as e:
+#             entry = InterviewExceptionEntry(e)
+#         return entry
+
+#     @property
+#     def traceback(self):
+#         """Return the exception as HTML."""
+#         e = self.exception
+#         tb_str = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+#         return tb_str
+
+
+#     @property
+#     def html(self):
+#         from rich.console import Console
+#         from rich.table import Table
+#         from rich.traceback import Traceback
+
+#         from io import StringIO
+#         html_output = StringIO()
+
+#         console = Console(file=html_output, record=True)
+#         tb = Traceback(show_locals=True)
+#         console.print(tb)
+
+#         tb = Traceback.from_exception(type(self.exception), self.exception, self.exception.__traceback__, show_locals=True)
+#         console.print(tb)
+#         return html_output.getvalue()
+
+#     def to_dict(self) -> dict:
+#         """Return the exception as a dictionary."""
+#         return {
+#             'exception': repr(self.exception),
+#             'time': self.time,
+#             'traceback': self.traceback
+#         }
 
 
 class InterviewExceptionCollection(UserDict):
@@ -84,3 +136,9 @@ class InterviewExceptionCollection(UserDict):
                 )
 
         console.print(table)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -13,6 +13,40 @@ from edsl.jobs.tasks.TaskHistory import TaskHistory
 from edsl.jobs.buckets.BucketCollection import BucketCollection
 from edsl.utilities.decorators import jupyter_nb_handler
 
+import time
+import functools
+
+
+def cache_with_timeout(timeout):
+    def decorator(func):
+        cached_result = {}
+        last_computation_time = [0]  # Using list to store mutable value
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            current_time = time.time()
+            if (current_time - last_computation_time[0]) >= timeout:
+                cached_result["value"] = func(*args, **kwargs)
+                last_computation_time[0] = current_time
+            return cached_result["value"]
+
+        return wrapper
+
+    return decorator
+
+
+# from queue import Queue
+from collections import UserList
+
+
+class StatusTracker(UserList):
+    def __init__(self, total_tasks: int):
+        self.total_tasks = total_tasks
+        super().__init__()
+
+    def current_status(self):
+        return print(f"Completed: {len(self.data)} of {self.total_tasks}", end="\r")
+
 
 class JobsRunnerAsyncio(JobsRunnerStatusMixin):
     """A class for running a collection of interviews asynchronously.
@@ -43,7 +77,9 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
 
         :param n: how many times to run each interview
         :param debug:
-        :param stop_on_exception:
+        :param stop_on_exception: Whether to stop the interview if an exception is raised
+        :param sidecar_model: a language model to use in addition to the interview's model
+        :param total_interviews: A list of interviews to run can be provided instead.
         """
         tasks = []
         if total_interviews:
@@ -87,15 +123,18 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
                     )  # set the cache for the first interview
                     self.total_interviews.append(interview)
 
-    async def run_async(self, cache=None) -> Results:
+    async def run_async(self, cache=None, n=1) -> Results:
         from edsl.results.Results import Results
 
+        # breakpoint()
+        # tracker = StatusTracker(total_tasks=len(self.interviews))
+
         if cache is None:
             self.cache = Cache()
         else:
             self.cache = cache
         data = []
-        async for result in self.run_async_generator(cache=self.cache):
+        async for result in self.run_async_generator(cache=self.cache, n=n):
             data.append(result)
         return Results(survey=self.jobs.survey, data=data)
 
@@ -173,6 +212,8 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
             raw_model_response=raw_model_results_dictionary,
             survey=interview.survey,
         )
+        result.interview_hash = hash(interview)
+
         return result
 
     @property
@@ -201,97 +242,86 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
         self.sidecar_model = sidecar_model
 
         from edsl.results.Results import Results
+        from rich.live import Live
+        from rich.console import Console
 
-        if not progress_bar:
-            # print("Running without progress bar")
-            with cache as c:
-
-                async def process_results():
-                    """Processes results from interviews."""
-                    async for result in self.run_async_generator(
-                        n=n,
-                        debug=debug,
-                        stop_on_exception=stop_on_exception,
-                        cache=c,
-                        sidecar_model=sidecar_model,
-                    ):
-                        self.results.append(result)
-                    self.completed = True
-
-                await asyncio.gather(process_results())
-
-            results = Results(survey=self.jobs.survey, data=self.results)
-        else:
-            # print("Running with progress bar")
-            from rich.live import Live
-            from rich.console import Console
-
-            def generate_table():
-                return self.status_table(self.results, self.elapsed_time)
+        @cache_with_timeout(1)
+        def generate_table():
+            return self.status_table(self.results, self.elapsed_time)
 
-            @contextmanager
-            def no_op_cm():
-                """A no-op context manager with a dummy update method."""
-                yield DummyLive()
+        async def process_results(cache, progress_bar_context=None):
+            """Processes results from interviews."""
+            async for result in self.run_async_generator(
+                n=n,
+                debug=debug,
+                stop_on_exception=stop_on_exception,
+                cache=cache,
+                sidecar_model=sidecar_model,
+            ):
+                self.results.append(result)
+                if progress_bar_context:
+                    progress_bar_context.update(generate_table())
+                self.completed = True
+
+        async def update_progress_bar(progress_bar_context):
+            """Updates the progress bar at fixed intervals."""
+            if progress_bar_context is None:
+                return
+
+            while True:
+                progress_bar_context.update(generate_table())
+                await asyncio.sleep(0.1)  # Update interval
+                if self.completed:
+                    break
+
+        @contextmanager
+        def conditional_context(condition, context_manager):
+            if condition:
+                with context_manager as cm:
+                    yield cm
+            else:
+                yield
+
+        with conditional_context(
+            progress_bar, Live(generate_table(), console=console, refresh_per_second=1)
+        ) as progress_bar_context:
+            with cache as c:
+                progress_task = asyncio.create_task(
+                    update_progress_bar(progress_bar_context)
+                )
 
-            class DummyLive:
-                def update(self, *args, **kwargs):
-                    """A dummy update method that does nothing."""
+                try:
+                    await asyncio.gather(
+                        progress_task,
+                        process_results(
+                            cache=c, progress_bar_context=progress_bar_context
+                        ),
+                    )
+                except asyncio.CancelledError:
                     pass
+                finally:
+                    progress_task.cancel()  # Cancel the progress_task when process_results is done
+                    await progress_task
 
-            progress_bar_context = (
-                Live(generate_table(), console=console, refresh_per_second=5)
-                if progress_bar
-                else no_op_cm()
-            )
+                    await asyncio.sleep(1)  # short delay to show the final status
 
-            with cache as c:
-                with progress_bar_context as live:
-
-                    async def update_progress_bar():
-                        """Updates the progress bar at fixed intervals."""
-                        while True:
-                            live.update(generate_table())
-                            await asyncio.sleep(0.00001)  # Update interval
-                            if self.completed:
-                                break
-
-                    async def process_results():
-                        """Processes results from interviews."""
-                        async for result in self.run_async_generator(
-                            n=n,
-                            debug=debug,
-                            stop_on_exception=stop_on_exception,
-                            cache=c,
-                            sidecar_model=sidecar_model,
-                        ):
-                            self.results.append(result)
-                            live.update(generate_table())
-                        self.completed = True
-
-                    progress_task = asyncio.create_task(update_progress_bar())
-
-                    try:
-                        await asyncio.gather(process_results(), progress_task)
-                    except asyncio.CancelledError:
-                        pass
-                    finally:
-                        progress_task.cancel()  # Cancel the progress_task when process_results is done
-                        await progress_task
-
-                        await asyncio.sleep(1)  # short delay to show the final status
-
-                        # one more update
-                        live.update(generate_table())
-
-            results = Results(survey=self.jobs.survey, data=self.results)
+                    if progress_bar_context:
+                        progress_bar_context.update(generate_table())
+
+        # puts results in the same order as the total interviews
+        interview_hashes = [hash(interview) for interview in self.total_interviews]
+        self.results = sorted(
+            self.results, key=lambda x: interview_hashes.index(x.interview_hash)
+        )
 
+        results = Results(survey=self.jobs.survey, data=self.results)
         task_history = TaskHistory(self.total_interviews, include_traceback=False)
         results.task_history = task_history
 
         results.has_exceptions = task_history.has_exceptions
 
         if results.has_exceptions:
+            # put the failed interviews in the results object as a list
             failed_interviews = [
                 interview.duplicate(
                     iteration=interview.iteration, cache=interview.cache
@@ -312,6 +342,7 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
 
                 shared_globals["edsl_runner_exceptions"] = task_history
                 print(msg)
+                # this is where exceptions are opening up
                 task_history.html(cta="Open report to see details.")
                 print(
                     "Also see: https://docs.expectedparrot.com/en/latest/exceptions.html"