edsl 0.1.30.dev4__py3-none-any.whl → 0.1.31__py3-none-any.whl

edsl/jobs/interviews/InterviewTaskBuildingMixin.py

@@ -12,20 +12,38 @@ from edsl.exceptions import InterviewTimeoutError
 # from edsl.questions.QuestionBase import QuestionBase
 from edsl.surveys.base import EndOfSurvey
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
-from edsl.jobs.interviews.interview_exception_tracking import InterviewExceptionEntry
+from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.tasks.task_status_enum import TaskStatus
 from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
 
 # from edsl.agents.InvigilatorBase import InvigilatorBase
 
+from rich.console import Console
+from rich.traceback import Traceback
+
 TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
 
 
+def frame_summary_to_dict(frame):
+    """
+    Convert a FrameSummary object to a dictionary.
+
+    :param frame: A traceback FrameSummary object
+    :return: A dictionary containing the frame's details
+    """
+    return {
+        "filename": frame.filename,
+        "lineno": frame.lineno,
+        "name": frame.name,
+        "line": frame.line,
+    }
+
+
 class InterviewTaskBuildingMixin:
     def _build_invigilators(
         self, debug: bool
-    ) -> Generator[InvigilatorBase, None, None]:
+    ) -> Generator["InvigilatorBase", None, None]:
         """Create an invigilator for each question.
 
         :param debug: whether to use debug mode, in which case `InvigilatorDebug` is used.
@@ -35,7 +53,7 @@ class InterviewTaskBuildingMixin:
         for question in self.survey.questions:
             yield self._get_invigilator(question=question, debug=debug)
 
-    def _get_invigilator(self, question: QuestionBase, debug: bool) -> "Invigilator":
+    def _get_invigilator(self, question: "QuestionBase", debug: bool) -> "Invigilator":
         """Return an invigilator for the given question.
 
         :param question: the question to be answered
@@ -84,7 +102,7 @@ class InterviewTaskBuildingMixin:
         return tuple(tasks)  # , invigilators
 
     def _get_tasks_that_must_be_completed_before(
-        self, *, tasks: list[asyncio.Task], question: QuestionBase
+        self, *, tasks: list[asyncio.Task], question: "QuestionBase"
     ) -> Generator[asyncio.Task, None, None]:
         """Return the tasks that must be completed before the given question can be answered.
 
@@ -100,7 +118,7 @@ class InterviewTaskBuildingMixin:
     def _create_question_task(
         self,
         *,
-        question: QuestionBase,
+        question: "QuestionBase",
         tasks_that_must_be_completed_before: list[asyncio.Task],
         model_buckets: ModelBuckets,
         debug: bool,
@@ -148,7 +166,6 @@ class InterviewTaskBuildingMixin:
                 raise ValueError(f"Prompt is of type {type(prompt)}")
         return len(combined_text) / 4.0
 
-    @retry_strategy
     async def _answer_question_and_record_task(
         self,
         *,
@@ -163,36 +180,33 @@ class InterviewTaskBuildingMixin:
         """
         from edsl.data_transfer_models import AgentResponseDict
 
-        try:
-            invigilator = self._get_invigilator(question, debug=debug)
+        async def _inner():
+            try:
+                invigilator = self._get_invigilator(question, debug=debug)
 
-            if self._skip_this_question(question):
-                return invigilator.get_failed_task_result()
+                if self._skip_this_question(question):
+                    return invigilator.get_failed_task_result()
 
-            response: AgentResponseDict = await self._attempt_to_answer_question(
-                invigilator, task
-            )
+                response: AgentResponseDict = await self._attempt_to_answer_question(
+                    invigilator, task
+                )
 
-            self._add_answer(response=response, question=question)
+                self._add_answer(response=response, question=question)
 
-            # With the answer to the question, we can now cancel any skipped questions
-            self._cancel_skipped_questions(question)
-            return AgentResponseDict(**response)
-        except Exception as e:
-            raise e
-            # import traceback
-            # print("Exception caught:")
-            # traceback.print_exc()
-
-            # # Extract and print the traceback info
-            # tb = e.__traceback__
-            # while tb is not None:
-            #     print(f"File {tb.tb_frame.f_code.co_filename}, line {tb.tb_lineno}, in {tb.tb_frame.f_code.co_name}")
-            #     tb = tb.tb_next
-            #     breakpoint()
-            #     raise e
-
-    def _add_answer(self, response: AgentResponseDict, question: QuestionBase) -> None:
+                self._cancel_skipped_questions(question)
+                return AgentResponseDict(**response)
+            except Exception as e:
+                raise e
+
+        skip_rety = getattr(self, "skip_retry", False)
+        if not skip_rety:
+            _inner = retry_strategy(_inner)
+
+        return await _inner()
+
+    def _add_answer(
+        self, response: "AgentResponseDict", question: "QuestionBase"
+    ) -> None:
         """Add the answer to the answers dictionary.
 
         :param response: the response to the question.
@@ -200,7 +214,7 @@ class InterviewTaskBuildingMixin:
         """
         self.answers.add_answer(response=response, question=question)
 
-    def _skip_this_question(self, current_question: QuestionBase) -> bool:
+    def _skip_this_question(self, current_question: "QuestionBase") -> bool:
         """Determine if the current question should be skipped.
 
         :param current_question: the question to be answered.
@@ -213,38 +227,30 @@ class InterviewTaskBuildingMixin:
         )
         return skip
 
+    def _handle_exception(self, e, question_name: str, task=None):
+        exception_entry = InterviewExceptionEntry(e)
+        if task:
+            task.task_status = TaskStatus.FAILED
+        self.exceptions.add(question_name, exception_entry)
+
     async def _attempt_to_answer_question(
-        self, invigilator: InvigilatorBase, task: asyncio.Task
-    ) -> AgentResponseDict:
+        self, invigilator: "InvigilatorBase", task: asyncio.Task
+    ) -> "AgentResponseDict":
         """Attempt to answer the question, and handle exceptions.
 
         :param invigilator: the invigilator that will answer the question.
         :param task: the task that is being run.
+
         """
         try:
             return await asyncio.wait_for(
                 invigilator.async_answer_question(), timeout=TIMEOUT
             )
         except asyncio.TimeoutError as e:
-            exception_entry = InterviewExceptionEntry(
-                exception=repr(e),
-                time=time.time(),
-                traceback=traceback.format_exc(),
-            )
-            if task:
-                task.task_status = TaskStatus.FAILED
-            self.exceptions.add(invigilator.question.question_name, exception_entry)
-
+            self._handle_exception(e, invigilator.question.question_name, task)
             raise InterviewTimeoutError(f"Task timed out after {TIMEOUT} seconds.")
         except Exception as e:
-            exception_entry = InterviewExceptionEntry(
-                exception=repr(e),
-                time=time.time(),
-                traceback=traceback.format_exc(),
-            )
-            if task:
-                task.task_status = TaskStatus.FAILED
-            self.exceptions.add(invigilator.question.question_name, exception_entry)
+            self._handle_exception(e, invigilator.question.question_name, task)
             raise e
 
     def _cancel_skipped_questions(self, current_question: QuestionBase) -> None:

edsl/jobs/interviews/interview_exception_tracking.py

@@ -1,18 +1,70 @@
-from rich.console import Console
-from rich.table import Table
+import traceback
+import datetime
+import time
 from collections import UserDict
 
+from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 
-class InterviewExceptionEntry(UserDict):
-    """Class to record an exception that occurred during the interview."""
+#                 #traceback=traceback.format_exc(),
+#                 #traceback = frame_summary_to_dict(traceback.extract_tb(e.__traceback__))
+#                 #traceback = [frame_summary_to_dict(f) for f in traceback.extract_tb(e.__traceback__)]
 
-    def __init__(self, exception, time, traceback):
-        data = {"exception": exception, "time": time, "traceback": traceback}
-        super().__init__(data)
+# class InterviewExceptionEntry:
+#     """Class to record an exception that occurred during the interview.
 
-    def to_dict(self) -> dict:
-        """Return the exception as a dictionary."""
-        return self.data
+#     >>> entry = InterviewExceptionEntry.example()
+#     >>> entry.to_dict()['exception']
+#     "ValueError('An error occurred.')"
+#     """
+
+#     def __init__(self, exception: Exception):
+#         self.time = datetime.datetime.now().isoformat()
+#         self.exception = exception
+
+#     def __getitem__(self, key):
+#         # Support dict-like access obj['a']
+#         return str(getattr(self, key))
+
+#     @classmethod
+#     def example(cls):
+#         try:
+#             raise ValueError("An error occurred.")
+#         except Exception as e:
+#             entry = InterviewExceptionEntry(e)
+#         return entry
+
+#     @property
+#     def traceback(self):
+#         """Return the exception as HTML."""
+#         e = self.exception
+#         tb_str = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+#         return tb_str
+
+
+#     @property
+#     def html(self):
+#         from rich.console import Console
+#         from rich.table import Table
+#         from rich.traceback import Traceback
+
+#         from io import StringIO
+#         html_output = StringIO()
+
+#         console = Console(file=html_output, record=True)
+#         tb = Traceback(show_locals=True)
+#         console.print(tb)
+
+#         tb = Traceback.from_exception(type(self.exception), self.exception, self.exception.__traceback__, show_locals=True)
+#         console.print(tb)
+#         return html_output.getvalue()
+
+#     def to_dict(self) -> dict:
+#         """Return the exception as a dictionary."""
+#         return {
+#             'exception': repr(self.exception),
+#             'time': self.time,
+#             'traceback': self.traceback
+#         }
 
 
 class InterviewExceptionCollection(UserDict):
@@ -84,3 +136,9 @@ class InterviewExceptionCollection(UserDict):
                 )
 
         console.print(table)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -13,6 +13,40 @@ from edsl.jobs.tasks.TaskHistory import TaskHistory
 from edsl.jobs.buckets.BucketCollection import BucketCollection
 from edsl.utilities.decorators import jupyter_nb_handler
 
+import time
+import functools
+
+
+def cache_with_timeout(timeout):
+    def decorator(func):
+        cached_result = {}
+        last_computation_time = [0]  # Using list to store mutable value
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            current_time = time.time()
+            if (current_time - last_computation_time[0]) >= timeout:
+                cached_result["value"] = func(*args, **kwargs)
+                last_computation_time[0] = current_time
+            return cached_result["value"]
+
+        return wrapper
+
+    return decorator
+
+
+# from queue import Queue
+from collections import UserList
+
+
+class StatusTracker(UserList):
+    def __init__(self, total_tasks: int):
+        self.total_tasks = total_tasks
+        super().__init__()
+
+    def current_status(self):
+        return print(f"Completed: {len(self.data)} of {self.total_tasks}", end="\r")
+
 
 class JobsRunnerAsyncio(JobsRunnerStatusMixin):
     """A class for running a collection of interviews asynchronously.
@@ -43,7 +77,9 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
 
         :param n: how many times to run each interview
         :param debug:
-        :param stop_on_exception:
+        :param stop_on_exception: Whether to stop the interview if an exception is raised
+        :param sidecar_model: a language model to use in addition to the interview's model
+        :param total_interviews: A list of interviews to run can be provided instead.
         """
         tasks = []
         if total_interviews:
@@ -87,15 +123,18 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
                     )  # set the cache for the first interview
                     self.total_interviews.append(interview)
 
-    async def run_async(self, cache=None) -> Results:
+    async def run_async(self, cache=None, n=1) -> Results:
         from edsl.results.Results import Results
 
+        # breakpoint()
+        # tracker = StatusTracker(total_tasks=len(self.interviews))
+
         if cache is None:
             self.cache = Cache()
         else:
             self.cache = cache
         data = []
-        async for result in self.run_async_generator(cache=self.cache):
+        async for result in self.run_async_generator(cache=self.cache, n=n):
             data.append(result)
         return Results(survey=self.jobs.survey, data=data)
 
@@ -173,6 +212,8 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
             raw_model_response=raw_model_results_dictionary,
             survey=interview.survey,
         )
+        result.interview_hash = hash(interview)
+
         return result
 
     @property
@@ -201,97 +242,86 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
         self.sidecar_model = sidecar_model
 
         from edsl.results.Results import Results
+        from rich.live import Live
+        from rich.console import Console
 
-        if not progress_bar:
-            # print("Running without progress bar")
-            with cache as c:
-
-                async def process_results():
-                    """Processes results from interviews."""
-                    async for result in self.run_async_generator(
-                        n=n,
-                        debug=debug,
-                        stop_on_exception=stop_on_exception,
-                        cache=c,
-                        sidecar_model=sidecar_model,
-                    ):
-                        self.results.append(result)
-                    self.completed = True
-
-                await asyncio.gather(process_results())
-
-            results = Results(survey=self.jobs.survey, data=self.results)
-        else:
-            # print("Running with progress bar")
-            from rich.live import Live
-            from rich.console import Console
-
-            def generate_table():
-                return self.status_table(self.results, self.elapsed_time)
+        @cache_with_timeout(1)
+        def generate_table():
+            return self.status_table(self.results, self.elapsed_time)
 
-            @contextmanager
-            def no_op_cm():
-                """A no-op context manager with a dummy update method."""
-                yield DummyLive()
+        async def process_results(cache, progress_bar_context=None):
+            """Processes results from interviews."""
+            async for result in self.run_async_generator(
+                n=n,
+                debug=debug,
+                stop_on_exception=stop_on_exception,
+                cache=cache,
+                sidecar_model=sidecar_model,
+            ):
+                self.results.append(result)
+                if progress_bar_context:
+                    progress_bar_context.update(generate_table())
+                self.completed = True
+
+        async def update_progress_bar(progress_bar_context):
+            """Updates the progress bar at fixed intervals."""
+            if progress_bar_context is None:
+                return
+
+            while True:
+                progress_bar_context.update(generate_table())
+                await asyncio.sleep(0.1)  # Update interval
+                if self.completed:
+                    break
+
+        @contextmanager
+        def conditional_context(condition, context_manager):
+            if condition:
+                with context_manager as cm:
+                    yield cm
+            else:
+                yield
+
+        with conditional_context(
+            progress_bar, Live(generate_table(), console=console, refresh_per_second=1)
+        ) as progress_bar_context:
+            with cache as c:
+                progress_task = asyncio.create_task(
+                    update_progress_bar(progress_bar_context)
+                )
 
-            class DummyLive:
-                def update(self, *args, **kwargs):
-                    """A dummy update method that does nothing."""
+                try:
+                    await asyncio.gather(
+                        progress_task,
+                        process_results(
+                            cache=c, progress_bar_context=progress_bar_context
+                        ),
+                    )
+                except asyncio.CancelledError:
                     pass
+                finally:
+                    progress_task.cancel()  # Cancel the progress_task when process_results is done
+                    await progress_task
 
-            progress_bar_context = (
-                Live(generate_table(), console=console, refresh_per_second=5)
-                if progress_bar
-                else no_op_cm()
-            )
+                    await asyncio.sleep(1)  # short delay to show the final status
 
-            with cache as c:
-                with progress_bar_context as live:
-
-                    async def update_progress_bar():
-                        """Updates the progress bar at fixed intervals."""
-                        while True:
-                            live.update(generate_table())
-                            await asyncio.sleep(0.00001)  # Update interval
-                            if self.completed:
-                                break
-
-                    async def process_results():
-                        """Processes results from interviews."""
-                        async for result in self.run_async_generator(
-                            n=n,
-                            debug=debug,
-                            stop_on_exception=stop_on_exception,
-                            cache=c,
-                            sidecar_model=sidecar_model,
-                        ):
-                            self.results.append(result)
-                            live.update(generate_table())
-                        self.completed = True
-
-                    progress_task = asyncio.create_task(update_progress_bar())
-
-                    try:
-                        await asyncio.gather(process_results(), progress_task)
-                    except asyncio.CancelledError:
-                        pass
-                    finally:
-                        progress_task.cancel()  # Cancel the progress_task when process_results is done
-                        await progress_task
-
-                        await asyncio.sleep(1)  # short delay to show the final status
-
-                        # one more update
-                        live.update(generate_table())
-
-            results = Results(survey=self.jobs.survey, data=self.results)
+                    if progress_bar_context:
+                        progress_bar_context.update(generate_table())
+
+        # puts results in the same order as the total interviews
+        interview_hashes = [hash(interview) for interview in self.total_interviews]
+        self.results = sorted(
+            self.results, key=lambda x: interview_hashes.index(x.interview_hash)
+        )
 
+        results = Results(survey=self.jobs.survey, data=self.results)
         task_history = TaskHistory(self.total_interviews, include_traceback=False)
         results.task_history = task_history
 
         results.has_exceptions = task_history.has_exceptions
 
         if results.has_exceptions:
+            # put the failed interviews in the results object as a list
             failed_interviews = [
                 interview.duplicate(
                     iteration=interview.iteration, cache=interview.cache
@@ -312,6 +342,7 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
 
                 shared_globals["edsl_runner_exceptions"] = task_history
                 print(msg)
+                # this is where exceptions are opening up
                 task_history.html(cta="Open report to see details.")
                 print(
                     "Also see: https://docs.expectedparrot.com/en/latest/exceptions.html"