edsl 0.1.31__py3-none-any.whl → 0.1.31.dev1__py3-none-any.whl

edsl/jobs/buckets/TokenBucket.py

@@ -100,9 +100,7 @@ class TokenBucket:
         available_tokens = min(self.capacity, self.tokens + refill_amount)
         return max(0, requested_tokens - available_tokens) / self.refill_rate
 
-    async def get_tokens(
-        self, amount: Union[int, float] = 1, cheat_bucket_capacity=True
-    ) -> None:
+    async def get_tokens(self, amount: Union[int, float] = 1) -> None:
         """Wait for the specified number of tokens to become available.
 
 
@@ -118,20 +116,14 @@ class TokenBucket:
         True
 
         >>> bucket = TokenBucket(bucket_name="test", bucket_type="test", capacity=10, refill_rate=1)
-        >>> asyncio.run(bucket.get_tokens(11, cheat_bucket_capacity=False))
+        >>> asyncio.run(bucket.get_tokens(11))
         Traceback (most recent call last):
         ...
         ValueError: Requested amount exceeds bucket capacity. Bucket capacity: 10, requested amount: 11. As the bucket never overflows, the requested amount will never be available.
-        >>> asyncio.run(bucket.get_tokens(11, cheat_bucket_capacity=True))
         """
         if amount > self.capacity:
-            if not cheat_bucket_capacity:
-                msg = f"Requested amount exceeds bucket capacity. Bucket capacity: {self.capacity}, requested amount: {amount}. As the bucket never overflows, the requested amount will never be available."
-                raise ValueError(msg)
-            else:
-                self.tokens = 0  # clear the bucket but let it go through
-                return
-
+            msg = f"Requested amount exceeds bucket capacity. Bucket capacity: {self.capacity}, requested amount: {amount}. As the bucket never overflows, the requested amount will never be available."
+            raise ValueError(msg)
         while self.tokens < amount:
             self.refill()
             await asyncio.sleep(0.01)  # Sleep briefly to prevent busy waiting

edsl/jobs/interviews/Interview.py

@@ -14,8 +14,8 @@ from edsl.jobs.tasks.TaskCreators import TaskCreators
 from edsl.jobs.interviews.InterviewStatusLog import InterviewStatusLog
 from edsl.jobs.interviews.interview_exception_tracking import (
     InterviewExceptionCollection,
+    InterviewExceptionEntry,
 )
-from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.interviews.InterviewTaskBuildingMixin import InterviewTaskBuildingMixin
 from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
@@ -44,7 +44,6 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         iteration: int = 0,
         cache: Optional["Cache"] = None,
         sidecar_model: Optional["LanguageModel"] = None,
-        skip_retry=False,
     ):
         """Initialize the Interview instance.
 
@@ -88,7 +87,6 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         self.task_creators = TaskCreators()  # tracks the task creators
         self.exceptions = InterviewExceptionCollection()
         self._task_status_log_dict = InterviewStatusLog()
-        self.skip_retry = skip_retry
 
         # dictionary mapping question names to their index in the survey.
         self.to_index = {
@@ -96,30 +94,6 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
             for index, question_name in enumerate(self.survey.question_names)
         }
 
-    def _to_dict(self, include_exceptions=False) -> dict[str, Any]:
-        """Return a dictionary representation of the Interview instance.
-        This is just for hashing purposes.
-
-        >>> i = Interview.example()
-        >>> hash(i)
-        1646262796627658719
-        """
-        d = {
-            "agent": self.agent._to_dict(),
-            "survey": self.survey._to_dict(),
-            "scenario": self.scenario._to_dict(),
-            "model": self.model._to_dict(),
-            "iteration": self.iteration,
-            "exceptions": {},
-        }
-        if include_exceptions:
-            d["exceptions"] = self.exceptions.to_dict()
-
-    def __hash__(self) -> int:
-        from edsl.utilities.utilities import dict_hash
-
-        return dict_hash(self._to_dict())
-
     async def async_conduct_interview(
         self,
         *,
@@ -160,7 +134,8 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         <BLANKLINE>
 
         >>> i.exceptions
-        {'q0': ...
+        {'q0': [{'exception': "Exception('This is a test error')", 'time': ..., 'traceback': ...
+
         >>> i = Interview.example()
         >>> result, _ = asyncio.run(i.async_conduct_interview(stop_on_exception = True))
         Traceback (most recent call last):
@@ -229,9 +204,13 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         {}
         >>> i._record_exception(i.tasks[0], Exception("An exception occurred."))
         >>> i.exceptions
-        {'q0': ...
+        {'q0': [{'exception': "Exception('An exception occurred.')", 'time': ..., 'traceback': 'NoneType: None\\n'}]}
         """
-        exception_entry = InterviewExceptionEntry(exception)
+        exception_entry = InterviewExceptionEntry(
+            exception=repr(exception),
+            time=time.time(),
+            traceback=traceback.format_exc(),
+        )
         self.exceptions.add(task.get_name(), exception_entry)
 
     @property
@@ -272,7 +251,6 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
             model=self.model,
             iteration=iteration,
             cache=cache,
-            skip_retry=self.skip_retry,
         )
 
     @classmethod

edsl/jobs/interviews/InterviewTaskBuildingMixin.py

@@ -12,34 +12,16 @@ from edsl.exceptions import InterviewTimeoutError
 # from edsl.questions.QuestionBase import QuestionBase
 from edsl.surveys.base import EndOfSurvey
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
-from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
+from edsl.jobs.interviews.interview_exception_tracking import InterviewExceptionEntry
 from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.tasks.task_status_enum import TaskStatus
 from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
 
 # from edsl.agents.InvigilatorBase import InvigilatorBase
 
-from rich.console import Console
-from rich.traceback import Traceback
-
 TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
 
 
-def frame_summary_to_dict(frame):
-    """
-    Convert a FrameSummary object to a dictionary.
-
-    :param frame: A traceback FrameSummary object
-    :return: A dictionary containing the frame's details
-    """
-    return {
-        "filename": frame.filename,
-        "lineno": frame.lineno,
-        "name": frame.name,
-        "line": frame.line,
-    }
-
-
 class InterviewTaskBuildingMixin:
     def _build_invigilators(
         self, debug: bool
@@ -166,6 +148,7 @@ class InterviewTaskBuildingMixin:
                 raise ValueError(f"Prompt is of type {type(prompt)}")
         return len(combined_text) / 4.0
 
+    @retry_strategy
     async def _answer_question_and_record_task(
         self,
         *,
@@ -180,29 +163,22 @@ class InterviewTaskBuildingMixin:
         """
         from edsl.data_transfer_models import AgentResponseDict
 
-        async def _inner():
-            try:
-                invigilator = self._get_invigilator(question, debug=debug)
-
-                if self._skip_this_question(question):
-                    return invigilator.get_failed_task_result()
-
-                response: AgentResponseDict = await self._attempt_to_answer_question(
-                    invigilator, task
-                )
+        try:
+            invigilator = self._get_invigilator(question, debug=debug)
 
-                self._add_answer(response=response, question=question)
+            if self._skip_this_question(question):
+                return invigilator.get_failed_task_result()
 
-                self._cancel_skipped_questions(question)
-                return AgentResponseDict(**response)
-            except Exception as e:
-                raise e
+            response: AgentResponseDict = await self._attempt_to_answer_question(
+                invigilator, task
+            )
 
-        skip_rety = getattr(self, "skip_retry", False)
-        if not skip_rety:
-            _inner = retry_strategy(_inner)
+            self._add_answer(response=response, question=question)
 
-        return await _inner()
+            self._cancel_skipped_questions(question)
+            return AgentResponseDict(**response)
+        except Exception as e:
+            raise e
 
     def _add_answer(
         self, response: "AgentResponseDict", question: "QuestionBase"
@@ -227,30 +203,38 @@ class InterviewTaskBuildingMixin:
         )
         return skip
 
-    def _handle_exception(self, e, question_name: str, task=None):
-        exception_entry = InterviewExceptionEntry(e)
-        if task:
-            task.task_status = TaskStatus.FAILED
-        self.exceptions.add(question_name, exception_entry)
-
     async def _attempt_to_answer_question(
-        self, invigilator: "InvigilatorBase", task: asyncio.Task
-    ) -> "AgentResponseDict":
+        self, invigilator: InvigilatorBase, task: asyncio.Task
+    ) -> AgentResponseDict:
         """Attempt to answer the question, and handle exceptions.
 
         :param invigilator: the invigilator that will answer the question.
         :param task: the task that is being run.
-
         """
         try:
             return await asyncio.wait_for(
                 invigilator.async_answer_question(), timeout=TIMEOUT
             )
         except asyncio.TimeoutError as e:
-            self._handle_exception(e, invigilator.question.question_name, task)
+            exception_entry = InterviewExceptionEntry(
+                exception=repr(e),
+                time=time.time(),
+                traceback=traceback.format_exc(),
+            )
+            if task:
+                task.task_status = TaskStatus.FAILED
+            self.exceptions.add(invigilator.question.question_name, exception_entry)
+
             raise InterviewTimeoutError(f"Task timed out after {TIMEOUT} seconds.")
         except Exception as e:
-            self._handle_exception(e, invigilator.question.question_name, task)
+            exception_entry = InterviewExceptionEntry(
+                exception=repr(e),
+                time=time.time(),
+                traceback=traceback.format_exc(),
+            )
+            if task:
+                task.task_status = TaskStatus.FAILED
+            self.exceptions.add(invigilator.question.question_name, exception_entry)
             raise e
 
     def _cancel_skipped_questions(self, current_question: QuestionBase) -> None:

edsl/jobs/interviews/interview_exception_tracking.py

@@ -1,70 +1,18 @@
-import traceback
-import datetime
-import time
+from rich.console import Console
+from rich.table import Table
 from collections import UserDict
 
-from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 
-#                 #traceback=traceback.format_exc(),
-#                 #traceback = frame_summary_to_dict(traceback.extract_tb(e.__traceback__))
-#                 #traceback = [frame_summary_to_dict(f) for f in traceback.extract_tb(e.__traceback__)]
+class InterviewExceptionEntry(UserDict):
+    """Class to record an exception that occurred during the interview."""
 
-# class InterviewExceptionEntry:
-#     """Class to record an exception that occurred during the interview.
+    def __init__(self, exception, time, traceback):
+        data = {"exception": exception, "time": time, "traceback": traceback}
+        super().__init__(data)
 
-#     >>> entry = InterviewExceptionEntry.example()
-#     >>> entry.to_dict()['exception']
-#     "ValueError('An error occurred.')"
-#     """
-
-#     def __init__(self, exception: Exception):
-#         self.time = datetime.datetime.now().isoformat()
-#         self.exception = exception
-
-#     def __getitem__(self, key):
-#         # Support dict-like access obj['a']
-#         return str(getattr(self, key))
-
-#     @classmethod
-#     def example(cls):
-#         try:
-#             raise ValueError("An error occurred.")
-#         except Exception as e:
-#             entry = InterviewExceptionEntry(e)
-#         return entry
-
-#     @property
-#     def traceback(self):
-#         """Return the exception as HTML."""
-#         e = self.exception
-#         tb_str = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
-#         return tb_str
-
-
-#     @property
-#     def html(self):
-#         from rich.console import Console
-#         from rich.table import Table
-#         from rich.traceback import Traceback
-
-#         from io import StringIO
-#         html_output = StringIO()
-
-#         console = Console(file=html_output, record=True)
-#         tb = Traceback(show_locals=True)
-#         console.print(tb)
-
-#         tb = Traceback.from_exception(type(self.exception), self.exception, self.exception.__traceback__, show_locals=True)
-#         console.print(tb)
-#         return html_output.getvalue()
-
-#     def to_dict(self) -> dict:
-#         """Return the exception as a dictionary."""
-#         return {
-#             'exception': repr(self.exception),
-#             'time': self.time,
-#             'traceback': self.traceback
-#         }
+    def to_dict(self) -> dict:
+        """Return the exception as a dictionary."""
+        return self.data
 
 
 class InterviewExceptionCollection(UserDict):
@@ -136,9 +84,3 @@ class InterviewExceptionCollection(UserDict):
                 )
 
         console.print(table)
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -13,40 +13,6 @@ from edsl.jobs.tasks.TaskHistory import TaskHistory
 from edsl.jobs.buckets.BucketCollection import BucketCollection
 from edsl.utilities.decorators import jupyter_nb_handler
 
-import time
-import functools
-
-
-def cache_with_timeout(timeout):
-    def decorator(func):
-        cached_result = {}
-        last_computation_time = [0]  # Using list to store mutable value
-
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            current_time = time.time()
-            if (current_time - last_computation_time[0]) >= timeout:
-                cached_result["value"] = func(*args, **kwargs)
-                last_computation_time[0] = current_time
-            return cached_result["value"]
-
-        return wrapper
-
-    return decorator
-
-
-# from queue import Queue
-from collections import UserList
-
-
-class StatusTracker(UserList):
-    def __init__(self, total_tasks: int):
-        self.total_tasks = total_tasks
-        super().__init__()
-
-    def current_status(self):
-        return print(f"Completed: {len(self.data)} of {self.total_tasks}", end="\r")
-
 
 class JobsRunnerAsyncio(JobsRunnerStatusMixin):
     """A class for running a collection of interviews asynchronously.
@@ -77,9 +43,7 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
 
         :param n: how many times to run each interview
         :param debug:
-        :param stop_on_exception: Whether to stop the interview if an exception is raised
-        :param sidecar_model: a language model to use in addition to the interview's model
-        :param total_interviews: A list of interviews to run can be provided instead.
+        :param stop_on_exception:
         """
         tasks = []
         if total_interviews:
@@ -123,18 +87,15 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
                     )  # set the cache for the first interview
                     self.total_interviews.append(interview)
 
-    async def run_async(self, cache=None, n=1) -> Results:
+    async def run_async(self, cache=None) -> Results:
         from edsl.results.Results import Results
 
-        # breakpoint()
-        # tracker = StatusTracker(total_tasks=len(self.interviews))
-
         if cache is None:
             self.cache = Cache()
         else:
             self.cache = cache
         data = []
-        async for result in self.run_async_generator(cache=self.cache, n=n):
+        async for result in self.run_async_generator(cache=self.cache):
             data.append(result)
         return Results(survey=self.jobs.survey, data=data)
 
@@ -212,8 +173,6 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
             raw_model_response=raw_model_results_dictionary,
             survey=interview.survey,
         )
-        result.interview_hash = hash(interview)
-
         return result
 
     @property
@@ -242,86 +201,97 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
         self.sidecar_model = sidecar_model
 
         from edsl.results.Results import Results
-        from rich.live import Live
-        from rich.console import Console
-
-        @cache_with_timeout(1)
-        def generate_table():
-            return self.status_table(self.results, self.elapsed_time)
 
-        async def process_results(cache, progress_bar_context=None):
-            """Processes results from interviews."""
-            async for result in self.run_async_generator(
-                n=n,
-                debug=debug,
-                stop_on_exception=stop_on_exception,
-                cache=cache,
-                sidecar_model=sidecar_model,
-            ):
-                self.results.append(result)
-                if progress_bar_context:
-                    progress_bar_context.update(generate_table())
-                self.completed = True
-
-        async def update_progress_bar(progress_bar_context):
-            """Updates the progress bar at fixed intervals."""
-            if progress_bar_context is None:
-                return
-
-            while True:
-                progress_bar_context.update(generate_table())
-                await asyncio.sleep(0.1)  # Update interval
-                if self.completed:
-                    break
-
-        @contextmanager
-        def conditional_context(condition, context_manager):
-            if condition:
-                with context_manager as cm:
-                    yield cm
-            else:
-                yield
-
-        with conditional_context(
-            progress_bar, Live(generate_table(), console=console, refresh_per_second=1)
-        ) as progress_bar_context:
+        if not progress_bar:
+            # print("Running without progress bar")
             with cache as c:
-                progress_task = asyncio.create_task(
-                    update_progress_bar(progress_bar_context)
-                )
 
-                try:
-                    await asyncio.gather(
-                        progress_task,
-                        process_results(
-                            cache=c, progress_bar_context=progress_bar_context
-                        ),
-                    )
-                except asyncio.CancelledError:
-                    pass
-                finally:
-                    progress_task.cancel()  # Cancel the progress_task when process_results is done
-                    await progress_task
+                async def process_results():
+                    """Processes results from interviews."""
+                    async for result in self.run_async_generator(
+                        n=n,
+                        debug=debug,
+                        stop_on_exception=stop_on_exception,
+                        cache=c,
+                        sidecar_model=sidecar_model,
+                    ):
+                        self.results.append(result)
+                    self.completed = True
+
+                await asyncio.gather(process_results())
+
+            results = Results(survey=self.jobs.survey, data=self.results)
+        else:
+            # print("Running with progress bar")
+            from rich.live import Live
+            from rich.console import Console
 
-                    await asyncio.sleep(1)  # short delay to show the final status
+            def generate_table():
+                return self.status_table(self.results, self.elapsed_time)
 
-                    if progress_bar_context:
-                        progress_bar_context.update(generate_table())
+            @contextmanager
+            def no_op_cm():
+                """A no-op context manager with a dummy update method."""
+                yield DummyLive()
 
-        # puts results in the same order as the total interviews
-        interview_hashes = [hash(interview) for interview in self.total_interviews]
-        self.results = sorted(
-            self.results, key=lambda x: interview_hashes.index(x.interview_hash)
-        )
+            class DummyLive:
+                def update(self, *args, **kwargs):
+                    """A dummy update method that does nothing."""
+                    pass
+
+            progress_bar_context = (
+                Live(generate_table(), console=console, refresh_per_second=5)
+                if progress_bar
+                else no_op_cm()
+            )
+
+            with cache as c:
+                with progress_bar_context as live:
+
+                    async def update_progress_bar():
+                        """Updates the progress bar at fixed intervals."""
+                        while True:
+                            live.update(generate_table())
+                            await asyncio.sleep(0.00001)  # Update interval
+                            if self.completed:
+                                break
+
+                    async def process_results():
+                        """Processes results from interviews."""
+                        async for result in self.run_async_generator(
+                            n=n,
+                            debug=debug,
+                            stop_on_exception=stop_on_exception,
+                            cache=c,
+                            sidecar_model=sidecar_model,
+                        ):
+                            self.results.append(result)
+                            live.update(generate_table())
+                        self.completed = True
+
+                    progress_task = asyncio.create_task(update_progress_bar())
+
+                    try:
+                        await asyncio.gather(process_results(), progress_task)
+                    except asyncio.CancelledError:
+                        pass
+                    finally:
+                        progress_task.cancel()  # Cancel the progress_task when process_results is done
+                        await progress_task
+
+                        await asyncio.sleep(1)  # short delay to show the final status
+
+                        # one more update
+                        live.update(generate_table())
+
+            results = Results(survey=self.jobs.survey, data=self.results)
 
-        results = Results(survey=self.jobs.survey, data=self.results)
         task_history = TaskHistory(self.total_interviews, include_traceback=False)
         results.task_history = task_history
 
         results.has_exceptions = task_history.has_exceptions
 
         if results.has_exceptions:
-            # put the failed interviews in the results object as a list
             failed_interviews = [
                 interview.duplicate(
                     iteration=interview.iteration, cache=interview.cache
@@ -342,7 +312,6 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
 
                 shared_globals["edsl_runner_exceptions"] = task_history
                 print(msg)
-                # this is where exceptions are opening up
                 task_history.html(cta="Open report to see details.")
                 print(
                     "Also see: https://docs.expectedparrot.com/en/latest/exceptions.html"