edsl 0.1.31.dev4__py3-none-any.whl → 0.1.32__py3-none-any.whl

edsl/jobs/interviews/InterviewTaskBuildingMixin.py

@@ -12,16 +12,34 @@ from edsl.exceptions import InterviewTimeoutError
 # from edsl.questions.QuestionBase import QuestionBase
 from edsl.surveys.base import EndOfSurvey
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
-from edsl.jobs.interviews.interview_exception_tracking import InterviewExceptionEntry
+from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.tasks.task_status_enum import TaskStatus
 from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
 
 # from edsl.agents.InvigilatorBase import InvigilatorBase
 
+from rich.console import Console
+from rich.traceback import Traceback
+
 TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
 
 
+def frame_summary_to_dict(frame):
+    """
+    Convert a FrameSummary object to a dictionary.
+
+    :param frame: A traceback FrameSummary object
+    :return: A dictionary containing the frame's details
+    """
+    return {
+        "filename": frame.filename,
+        "lineno": frame.lineno,
+        "name": frame.name,
+        "line": frame.line,
+    }
+
+
 class InterviewTaskBuildingMixin:
     def _build_invigilators(
         self, debug: bool
@@ -148,7 +166,6 @@ class InterviewTaskBuildingMixin:
                 raise ValueError(f"Prompt is of type {type(prompt)}")
         return len(combined_text) / 4.0
 
-    @retry_strategy
     async def _answer_question_and_record_task(
         self,
         *,
@@ -163,22 +180,29 @@ class InterviewTaskBuildingMixin:
         """
         from edsl.data_transfer_models import AgentResponseDict
 
-        try:
-            invigilator = self._get_invigilator(question, debug=debug)
+        async def _inner():
+            try:
+                invigilator = self._get_invigilator(question, debug=debug)
 
-            if self._skip_this_question(question):
-                return invigilator.get_failed_task_result()
+                if self._skip_this_question(question):
+                    return invigilator.get_failed_task_result()
 
-            response: AgentResponseDict = await self._attempt_to_answer_question(
-                invigilator, task
-            )
+                response: AgentResponseDict = await self._attempt_to_answer_question(
+                    invigilator, task
+                )
 
-            self._add_answer(response=response, question=question)
+                self._add_answer(response=response, question=question)
 
-            self._cancel_skipped_questions(question)
-            return AgentResponseDict(**response)
-        except Exception as e:
-            raise e
+                self._cancel_skipped_questions(question)
+                return AgentResponseDict(**response)
+            except Exception as e:
+                raise e
+
+        skip_rety = getattr(self, "skip_retry", False)
+        if not skip_rety:
+            _inner = retry_strategy(_inner)
+
+        return await _inner()
 
     def _add_answer(
         self, response: "AgentResponseDict", question: "QuestionBase"
@@ -203,39 +227,30 @@ class InterviewTaskBuildingMixin:
         )
         return skip
 
+    def _handle_exception(self, e, question_name: str, task=None):
+        exception_entry = InterviewExceptionEntry(e)
+        if task:
+            task.task_status = TaskStatus.FAILED
+        self.exceptions.add(question_name, exception_entry)
+
     async def _attempt_to_answer_question(
-        self, invigilator: 'InvigilatorBase', task: asyncio.Task
-    ) -> 'AgentResponseDict':
+        self, invigilator: "InvigilatorBase", task: asyncio.Task
+    ) -> "AgentResponseDict":
         """Attempt to answer the question, and handle exceptions.
 
         :param invigilator: the invigilator that will answer the question.
         :param task: the task that is being run.
-        
+
         """
         try:
             return await asyncio.wait_for(
                 invigilator.async_answer_question(), timeout=TIMEOUT
             )
         except asyncio.TimeoutError as e:
-            exception_entry = InterviewExceptionEntry(
-                exception=repr(e),
-                time=time.time(),
-                traceback=traceback.format_exc(),
-            )
-            if task:
-                task.task_status = TaskStatus.FAILED
-            self.exceptions.add(invigilator.question.question_name, exception_entry)
-
+            self._handle_exception(e, invigilator.question.question_name, task)
             raise InterviewTimeoutError(f"Task timed out after {TIMEOUT} seconds.")
         except Exception as e:
-            exception_entry = InterviewExceptionEntry(
-                exception=repr(e),
-                time=time.time(),
-                traceback=traceback.format_exc(),
-            )
-            if task:
-                task.task_status = TaskStatus.FAILED
-            self.exceptions.add(invigilator.question.question_name, exception_entry)
+            self._handle_exception(e, invigilator.question.question_name, task)
             raise e
 
     def _cancel_skipped_questions(self, current_question: QuestionBase) -> None:

edsl/jobs/interviews/interview_exception_tracking.py

@@ -1,18 +1,70 @@
-from rich.console import Console
-from rich.table import Table
+import traceback
+import datetime
+import time
 from collections import UserDict
 
+from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 
-class InterviewExceptionEntry(UserDict):
-    """Class to record an exception that occurred during the interview."""
+#                 #traceback=traceback.format_exc(),
+#                 #traceback = frame_summary_to_dict(traceback.extract_tb(e.__traceback__))
+#                 #traceback = [frame_summary_to_dict(f) for f in traceback.extract_tb(e.__traceback__)]
 
-    def __init__(self, exception, time, traceback):
-        data = {"exception": exception, "time": time, "traceback": traceback}
-        super().__init__(data)
+# class InterviewExceptionEntry:
+#     """Class to record an exception that occurred during the interview.
 
-    def to_dict(self) -> dict:
-        """Return the exception as a dictionary."""
-        return self.data
+#     >>> entry = InterviewExceptionEntry.example()
+#     >>> entry.to_dict()['exception']
+#     "ValueError('An error occurred.')"
+#     """
+
+#     def __init__(self, exception: Exception):
+#         self.time = datetime.datetime.now().isoformat()
+#         self.exception = exception
+
+#     def __getitem__(self, key):
+#         # Support dict-like access obj['a']
+#         return str(getattr(self, key))
+
+#     @classmethod
+#     def example(cls):
+#         try:
+#             raise ValueError("An error occurred.")
+#         except Exception as e:
+#             entry = InterviewExceptionEntry(e)
+#         return entry
+
+#     @property
+#     def traceback(self):
+#         """Return the exception as HTML."""
+#         e = self.exception
+#         tb_str = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+#         return tb_str
+
+
+#     @property
+#     def html(self):
+#         from rich.console import Console
+#         from rich.table import Table
+#         from rich.traceback import Traceback
+
+#         from io import StringIO
+#         html_output = StringIO()
+
+#         console = Console(file=html_output, record=True)
+#         tb = Traceback(show_locals=True)
+#         console.print(tb)
+
+#         tb = Traceback.from_exception(type(self.exception), self.exception, self.exception.__traceback__, show_locals=True)
+#         console.print(tb)
+#         return html_output.getvalue()
+
+#     def to_dict(self) -> dict:
+#         """Return the exception as a dictionary."""
+#         return {
+#             'exception': repr(self.exception),
+#             'time': self.time,
+#             'traceback': self.traceback
+#         }
 
 
 class InterviewExceptionCollection(UserDict):
@@ -84,3 +136,9 @@ class InterviewExceptionCollection(UserDict):
                 )
 
         console.print(table)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -16,6 +16,7 @@ from edsl.utilities.decorators import jupyter_nb_handler
 import time
 import functools
 
+
 def cache_with_timeout(timeout):
     def decorator(func):
         cached_result = {}
@@ -25,23 +26,27 @@ def cache_with_timeout(timeout):
         def wrapper(*args, **kwargs):
             current_time = time.time()
             if (current_time - last_computation_time[0]) >= timeout:
-                cached_result['value'] = func(*args, **kwargs)
+                cached_result["value"] = func(*args, **kwargs)
                 last_computation_time[0] = current_time
-            return cached_result['value']
-        
+            return cached_result["value"]
+
         return wrapper
+
     return decorator
 
-#from queue import Queue
+
+# from queue import Queue
 from collections import UserList
 
+
 class StatusTracker(UserList):
     def __init__(self, total_tasks: int):
         self.total_tasks = total_tasks
         super().__init__()
-    
+
     def current_status(self):
-        return print(f"Completed: {len(self.data)} of {self.total_tasks}", end = "\r")
+        return print(f"Completed: {len(self.data)} of {self.total_tasks}", end="\r")
+
 
 class JobsRunnerAsyncio(JobsRunnerStatusMixin):
     """A class for running a collection of interviews asynchronously.
@@ -121,8 +126,8 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
     async def run_async(self, cache=None, n=1) -> Results:
         from edsl.results.Results import Results
 
-        #breakpoint()
-        #tracker = StatusTracker(total_tasks=len(self.interviews))
+        # breakpoint()
+        # tracker = StatusTracker(total_tasks=len(self.interviews))
 
         if cache is None:
             self.cache = Cache()
@@ -207,6 +212,8 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
             raw_model_response=raw_model_results_dictionary,
             survey=interview.survey,
         )
+        result.interview_hash = hash(interview)
+
         return result
 
     @property
@@ -242,7 +249,7 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
         def generate_table():
             return self.status_table(self.results, self.elapsed_time)
 
-        async def process_results(cache, progress_bar_context = None):
+        async def process_results(cache, progress_bar_context=None):
             """Processes results from interviews."""
             async for result in self.run_async_generator(
                 n=n,
@@ -275,16 +282,23 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
             else:
                 yield
 
-        with conditional_context(progress_bar, Live(generate_table(), console=console, refresh_per_second=1)) as progress_bar_context:
-
+        with conditional_context(
+            progress_bar, Live(generate_table(), console=console, refresh_per_second=1)
+        ) as progress_bar_context:
             with cache as c:
-
-                progress_task = asyncio.create_task(update_progress_bar(progress_bar_context))
+                progress_task = asyncio.create_task(
+                    update_progress_bar(progress_bar_context)
+                )
 
                 try:
-                    await asyncio.gather(progress_task, process_results(cache = c, progress_bar_context = progress_bar_context))
+                    await asyncio.gather(
+                        progress_task,
+                        process_results(
+                            cache=c, progress_bar_context=progress_bar_context
+                        ),
+                    )
                 except asyncio.CancelledError:
-                        pass
+                    pass
                 finally:
                     progress_task.cancel()  # Cancel the progress_task when process_results is done
                     await progress_task
@@ -294,6 +308,11 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
                     if progress_bar_context:
                         progress_bar_context.update(generate_table())
 
+        # puts results in the same order as the total interviews
+        interview_hashes = [hash(interview) for interview in self.total_interviews]
+        self.results = sorted(
+            self.results, key=lambda x: interview_hashes.index(x.interview_hash)
+        )
 
         results = Results(survey=self.jobs.survey, data=self.results)
         task_history = TaskHistory(self.total_interviews, include_traceback=False)
@@ -302,6 +321,7 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
         results.has_exceptions = task_history.has_exceptions
 
         if results.has_exceptions:
+            # put the failed interviews in the results object as a list
             failed_interviews = [
                 interview.duplicate(
                     iteration=interview.iteration, cache=interview.cache
@@ -322,6 +342,7 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
 
                 shared_globals["edsl_runner_exceptions"] = task_history
                 print(msg)
+                # this is where exceptions are opening up
                 task_history.html(cta="Open report to see details.")
                 print(
                     "Also see: https://docs.expectedparrot.com/en/latest/exceptions.html"

edsl/jobs/runners/JobsRunnerStatusMixin.py

@@ -22,7 +22,7 @@ from edsl.jobs.interviews.InterviewStatisticsCollection import (
 from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
 
 
-#return {"cache_status": token_usage_type, "details": details, "cost": f"${token_usage.cost(prices):.5f}"}
+# return {"cache_status": token_usage_type, "details": details, "cost": f"${token_usage.cost(prices):.5f}"}
 
 from dataclasses import dataclass, asdict
 
@@ -30,6 +30,7 @@ from rich.text import Text
 from rich.box import SIMPLE
 from rich.table import Table
 
+
 @dataclass
 class ModelInfo:
     model_name: str
@@ -45,18 +46,13 @@ class ModelTokenUsageStats:
     details: List[dict]
     cost: str
 
-class Stats:
 
+class Stats:
     def elapsed_time(self):
-        InterviewStatistic(
-                "elapsed_time", value=elapsed_time, digits=1, units="sec."
-            )
-
-
+        InterviewStatistic("elapsed_time", value=elapsed_time, digits=1, units="sec.")
 
 
 class JobsRunnerStatusMixin:
-
     # @staticmethod
     # def status_dict(interviews: List[Type["Interview"]]) -> List[Type[InterviewStatusDictionary]]:
     #     """
@@ -68,7 +64,6 @@ class JobsRunnerStatusMixin:
     #     return [interview.interview_status for interview in interviews]
 
     def _compute_statistic(stat_name: str, completed_tasks, elapsed_time, interviews):
-
         stat_definitions = {
             "elapsed_time": lambda: InterviewStatistic(
                 "elapsed_time", value=elapsed_time, digits=1, units="sec."
@@ -101,36 +96,49 @@ class JobsRunnerStatusMixin:
             "estimated_time_remaining": lambda: InterviewStatistic(
                 "estimated_time_remaining",
                 value=(
-                    (len(interviews) - len(completed_tasks)) * (elapsed_time / len(completed_tasks))
+                    (len(interviews) - len(completed_tasks))
+                    * (elapsed_time / len(completed_tasks))
                     if len(completed_tasks) > 0
                     else "NA"
                 ),
                 digits=1,
                 units="sec.",
-            )
-            }
+            ),
+        }
         if stat_name not in stat_definitions:
-            raise ValueError(f"Invalid stat_name: {stat_name}. The valid stat_names are: {list(stat_definitions.keys())}")
+            raise ValueError(
+                f"Invalid stat_name: {stat_name}. The valid stat_names are: {list(stat_definitions.keys())}"
+            )
         return stat_definitions[stat_name]()
 
-
     @staticmethod
-    def _job_level_info(completed_tasks: List[Type[asyncio.Task]], 
-                        elapsed_time: float, 
-                        interviews: List[Type["Interview"]] 
-                        ) -> InterviewStatisticsCollection:
-
+    def _job_level_info(
+        completed_tasks: List[Type[asyncio.Task]],
+        elapsed_time: float,
+        interviews: List[Type["Interview"]],
+    ) -> InterviewStatisticsCollection:
         interview_statistics = InterviewStatisticsCollection()
 
-        default_statistics = ["elapsed_time", "total_interviews_requested", "completed_interviews", "percent_complete", "average_time_per_interview", "task_remaining", "estimated_time_remaining"]
+        default_statistics = [
+            "elapsed_time",
+            "total_interviews_requested",
+            "completed_interviews",
+            "percent_complete",
+            "average_time_per_interview",
+            "task_remaining",
+            "estimated_time_remaining",
+        ]
         for stat_name in default_statistics:
-            interview_statistics.add_stat(JobsRunnerStatusMixin._compute_statistic(stat_name, completed_tasks, elapsed_time, interviews))
+            interview_statistics.add_stat(
+                JobsRunnerStatusMixin._compute_statistic(
+                    stat_name, completed_tasks, elapsed_time, interviews
+                )
+            )
 
         return interview_statistics
 
     @staticmethod
     def _get_model_queues_info(interviews):
-
         models_to_tokens = defaultdict(InterviewTokenUsage)
         model_to_status = defaultdict(InterviewStatusDictionary)
         waiting_dict = defaultdict(int)
@@ -141,14 +149,16 @@ class JobsRunnerStatusMixin:
             waiting_dict[interview.model] += interview.interview_status.waiting
 
         for model, num_waiting in waiting_dict.items():
-            yield JobsRunnerStatusMixin._get_model_info(model, num_waiting, models_to_tokens)
+            yield JobsRunnerStatusMixin._get_model_info(
+                model, num_waiting, models_to_tokens
+            )
 
     @staticmethod
     def generate_status_summary(
         completed_tasks: List[Type[asyncio.Task]],
         elapsed_time: float,
         interviews: List[Type["Interview"]],
-        include_model_queues = False
+        include_model_queues=False,
     ) -> InterviewStatisticsCollection:
         """Generate a summary of the status of the job runner.
 
@@ -164,13 +174,17 @@ class JobsRunnerStatusMixin:
         {'Elapsed time': '0.0 sec.', 'Total interviews requested': '1 ', 'Completed interviews': '0 ', 'Percent complete': '0 %', 'Average time per interview': 'NA', 'Task remaining': '1 ', 'Estimated time remaining': 'NA'}
         """
 
-        interview_status_summary: InterviewStatisticsCollection = JobsRunnerStatusMixin._job_level_info(
-            completed_tasks=completed_tasks, 
-            elapsed_time=elapsed_time, 
-            interviews=interviews
+        interview_status_summary: InterviewStatisticsCollection = (
+            JobsRunnerStatusMixin._job_level_info(
+                completed_tasks=completed_tasks,
+                elapsed_time=elapsed_time,
+                interviews=interviews,
+            )
         )
         if include_model_queues:
-            interview_status_summary.model_queues = list(JobsRunnerStatusMixin._get_model_queues_info(interviews))
+            interview_status_summary.model_queues = list(
+                JobsRunnerStatusMixin._get_model_queues_info(interviews)
+            )
         else:
             interview_status_summary.model_queues = None
 
@@ -202,15 +216,21 @@ class JobsRunnerStatusMixin:
 
         token_usage_info = []
         for token_usage_type in ["new_token_usage", "cached_token_usage"]:
-            token_usage_info.append(JobsRunnerStatusMixin._get_token_usage_info(token_usage_type, models_to_tokens, model, prices))
+            token_usage_info.append(
+                JobsRunnerStatusMixin._get_token_usage_info(
+                    token_usage_type, models_to_tokens, model, prices
+                )
+            )
 
-        return ModelInfo(**{
-            "model_name": model.model,
-            "TPM_limit_k": model.TPM / 1000,
-            "RPM_limit_k": model.RPM / 1000,
-            "num_tasks_waiting": num_waiting,
-            "token_usage_info": token_usage_info,
-        })
+        return ModelInfo(
+            **{
+                "model_name": model.model,
+                "TPM_limit_k": model.TPM / 1000,
+                "RPM_limit_k": model.RPM / 1000,
+                "num_tasks_waiting": num_waiting,
+                "token_usage_info": token_usage_info,
+            }
+        )
 
     @staticmethod
     def _get_token_usage_info(
@@ -232,13 +252,19 @@ class JobsRunnerStatusMixin:
 
         """
         all_token_usage: InterviewTokenUsage = models_to_tokens[model]
-        token_usage: TokenUsage = getattr(all_token_usage, token_usage_type)    
-        
-        details = [{"type": token_type, "tokens": getattr(token_usage, token_type)}
-                   for token_type in ["prompt_tokens", "completion_tokens"]]
-        
-        return ModelTokenUsageStats(token_usage_type = token_usage_type, details = details, cost = f"${token_usage.cost(prices):.5f}")
-    
+        token_usage: TokenUsage = getattr(all_token_usage, token_usage_type)
+
+        details = [
+            {"type": token_type, "tokens": getattr(token_usage, token_type)}
+            for token_type in ["prompt_tokens", "completion_tokens"]
+        ]
+
+        return ModelTokenUsageStats(
+            token_usage_type=token_usage_type,
+            details=details,
+            cost=f"${token_usage.cost(prices):.5f}",
+        )
+
     @staticmethod
     def _add_statistics_to_table(table, status_summary):
         table.add_column("Statistic", style="dim", no_wrap=True, width=50)
@@ -249,9 +275,7 @@ class JobsRunnerStatusMixin:
                 table.add_row(key, value)
 
     @staticmethod
-    def display_status_table(status_summary: InterviewStatisticsCollection) -> 'Table':
-
-
+    def display_status_table(status_summary: InterviewStatisticsCollection) -> "Table":
         table = Table(
             title="Job Status",
             show_header=True,
@@ -268,24 +292,29 @@ class JobsRunnerStatusMixin:
         if status_summary.model_queues is not None:
             table.add_row(Text("Model Queues", style="bold red"), "")
             for model_info in status_summary.model_queues:
-
                 model_name = model_info.model_name
                 tpm = f"TPM (k)={model_info.TPM_limit_k}"
                 rpm = f"RPM (k)= {model_info.RPM_limit_k}"
                 pretty_model_name = model_name + ";" + tpm + ";" + rpm
                 table.add_row(Text(pretty_model_name, style="blue"), "")
-                table.add_row("Number question tasks waiting for capacity", str(model_info.num_tasks_waiting))
+                table.add_row(
+                    "Number question tasks waiting for capacity",
+                    str(model_info.num_tasks_waiting),
+                )
                 # Token usage and cost info
                 for token_usage_info in model_info.token_usage_info:
                     token_usage_type = token_usage_info.token_usage_type
                     table.add_row(
-                        Text(spacing + token_usage_type.replace("_", " "), style="bold"), ""
+                        Text(
+                            spacing + token_usage_type.replace("_", " "), style="bold"
+                        ),
+                        "",
                     )
                     for detail in token_usage_info.details:
                         token_type = detail["type"]
                         tokens = detail["tokens"]
                         table.add_row(spacing + f"{token_type}", f"{tokens:,}")
-                    #table.add_row(spacing + "cost", cache_info["cost"])
+                    # table.add_row(spacing + "cost", cache_info["cost"])
 
         return table
 
@@ -297,6 +326,7 @@ class JobsRunnerStatusMixin:
         )
         return self.display_status_table(summary_data)
 
+
 if __name__ == "__main__":
     import doctest
 

edsl/jobs/tasks/TaskCreators.py

@@ -22,7 +22,7 @@ class TaskCreators(UserDict):
         This is iterates through all tasks that make up an interview.
         For each task, it determines how many tokens were used and whether they were cached or new.
         It then sums the total number of cached and new tokens used for the interview.
-        
+
         """
         cached_tokens = TokenUsage(from_cache=True)
         new_tokens = TokenUsage(from_cache=False)