edsl 0.1.33__py3-none-any.whl → 0.1.33.dev1__py3-none-any.whl

edsl/jobs/interviews/InterviewTaskBuildingMixin.py

@@ -0,0 +1,286 @@
+"""This module contains the Interview class, which is responsible for conducting an interview asynchronously."""
+
+from __future__ import annotations
+import asyncio
+import time
+import traceback
+from typing import Generator, Union
+
+from edsl import CONFIG
+from edsl.exceptions import InterviewTimeoutError
+
+# from edsl.questions.QuestionBase import QuestionBase
+from edsl.surveys.base import EndOfSurvey
+from edsl.jobs.buckets.ModelBuckets import ModelBuckets
+from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
+from edsl.jobs.interviews.retry_management import retry_strategy
+from edsl.jobs.tasks.task_status_enum import TaskStatus
+from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
+
+# from edsl.agents.InvigilatorBase import InvigilatorBase
+
+from rich.console import Console
+from rich.traceback import Traceback
+
+TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
+
+
+def frame_summary_to_dict(frame):
+    """
+    Convert a FrameSummary object to a dictionary.
+
+    :param frame: A traceback FrameSummary object
+    :return: A dictionary containing the frame's details
+    """
+    return {
+        "filename": frame.filename,
+        "lineno": frame.lineno,
+        "name": frame.name,
+        "line": frame.line,
+    }
+
+
+class InterviewTaskBuildingMixin:
+    def _build_invigilators(
+        self, debug: bool
+    ) -> Generator["InvigilatorBase", None, None]:
+        """Create an invigilator for each question.
+
+        :param debug: whether to use debug mode, in which case `InvigilatorDebug` is used.
+
+        An invigilator is responsible for answering a particular question in the survey.
+        """
+        for question in self.survey.questions:
+            yield self._get_invigilator(question=question, debug=debug)
+
+    def _get_invigilator(self, question: "QuestionBase", debug: bool) -> "Invigilator":
+        """Return an invigilator for the given question.
+
+        :param question: the question to be answered
+        :param debug: whether to use debug mode, in which case `InvigilatorDebug` is used.
+        """
+        invigilator = self.agent.create_invigilator(
+            question=question,
+            scenario=self.scenario,
+            model=self.model,
+            debug=debug,
+            survey=self.survey,
+            memory_plan=self.survey.memory_plan,
+            current_answers=self.answers,
+            iteration=self.iteration,
+            cache=self.cache,
+            sidecar_model=self.sidecar_model,
+        )
+        """Return an invigilator for the given question."""
+        return invigilator
+
+    def _build_question_tasks(
+        self,
+        debug: bool,
+        model_buckets: ModelBuckets,
+    ) -> list[asyncio.Task]:
+        """Create a task for each question, with dependencies on the questions that must be answered before this one can be answered.
+
+        :param debug: whether to use debug mode, in which case `InvigilatorDebug` is used.
+        :param model_buckets: the model buckets used to track and control usage rates.
+        """
+        tasks = []
+        for question in self.survey.questions:
+            tasks_that_must_be_completed_before = list(
+                self._get_tasks_that_must_be_completed_before(
+                    tasks=tasks, question=question
+                )
+            )
+            question_task = self._create_question_task(
+                question=question,
+                tasks_that_must_be_completed_before=tasks_that_must_be_completed_before,
+                model_buckets=model_buckets,
+                debug=debug,
+                iteration=self.iteration,
+            )
+            tasks.append(question_task)
+        return tuple(tasks)  # , invigilators
+
+    def _get_tasks_that_must_be_completed_before(
+        self, *, tasks: list[asyncio.Task], question: "QuestionBase"
+    ) -> Generator[asyncio.Task, None, None]:
+        """Return the tasks that must be completed before the given question can be answered.
+
+        :param tasks: a list of tasks that have been created so far.
+        :param question: the question for which we are determining dependencies.
+
+        If a question has no dependencies, this will be an empty list, [].
+        """
+        parents_of_focal_question = self.dag.get(question.question_name, [])
+        for parent_question_name in parents_of_focal_question:
+            yield tasks[self.to_index[parent_question_name]]
+
+    def _create_question_task(
+        self,
+        *,
+        question: "QuestionBase",
+        tasks_that_must_be_completed_before: list[asyncio.Task],
+        model_buckets: ModelBuckets,
+        debug: bool,
+        iteration: int = 0,
+    ) -> asyncio.Task:
+        """Create a task that depends on the passed-in dependencies that are awaited before the task is run.
+
+        :param question: the question to be answered. This is the question we are creating a task for.
+        :param tasks_that_must_be_completed_before: the tasks that must be completed before the focal task is run.
+        :param model_buckets: the model buckets used to track and control usage rates.
+        :param debug: whether to use debug mode, in which case `InvigilatorDebug` is used.
+        :param iteration: the iteration number for the interview.
+
+        The task is created by a `QuestionTaskCreator`, which is responsible for creating the task and managing its dependencies.
+        It is passed a reference to the function that will be called to answer the question.
+        It is passed a list "tasks_that_must_be_completed_before" that are awaited before the task is run.
+        These are added as a dependency to the focal task.
+        """
+        task_creator = QuestionTaskCreator(
+            question=question,
+            answer_question_func=self._answer_question_and_record_task,
+            token_estimator=self._get_estimated_request_tokens,
+            model_buckets=model_buckets,
+            iteration=iteration,
+        )
+        for task in tasks_that_must_be_completed_before:
+            task_creator.add_dependency(task)
+
+        self.task_creators.update(
+            {question.question_name: task_creator}
+        )  # track this task creator
+        return task_creator.generate_task(debug)
+
+    def _get_estimated_request_tokens(self, question) -> float:
+        """Estimate the number of tokens that will be required to run the focal task."""
+        invigilator = self._get_invigilator(question=question, debug=False)
+        # TODO: There should be a way to get a more accurate estimate.
+        combined_text = ""
+        for prompt in invigilator.get_prompts().values():
+            if hasattr(prompt, "text"):
+                combined_text += prompt.text
+            elif isinstance(prompt, str):
+                combined_text += prompt
+            else:
+                raise ValueError(f"Prompt is of type {type(prompt)}")
+        return len(combined_text) / 4.0
+
+    async def _answer_question_and_record_task(
+        self,
+        *,
+        question: "QuestionBase",
+        debug: bool,
+        task=None,
+    ) -> "AgentResponseDict":
+        """Answer a question and records the task.
+
+        This in turn calls the the passed-in agent's async_answer_question method, which returns a response dictionary.
+        Note that is updates answers dictionary with the response.
+        """
+        from edsl.data_transfer_models import AgentResponseDict
+
+        async def _inner():
+            try:
+                invigilator = self._get_invigilator(question, debug=debug)
+
+                if self._skip_this_question(question):
+                    return invigilator.get_failed_task_result()
+
+                response: AgentResponseDict = await self._attempt_to_answer_question(
+                    invigilator, task
+                )
+
+                self._add_answer(response=response, question=question)
+
+                self._cancel_skipped_questions(question)
+                return AgentResponseDict(**response)
+            except Exception as e:
+                raise e
+
+        skip_rety = getattr(self, "skip_retry", False)
+        if not skip_rety:
+            _inner = retry_strategy(_inner)
+
+        return await _inner()
+
+    def _add_answer(
+        self, response: "AgentResponseDict", question: "QuestionBase"
+    ) -> None:
+        """Add the answer to the answers dictionary.
+
+        :param response: the response to the question.
+        :param question: the question that was answered.
+        """
+        self.answers.add_answer(response=response, question=question)
+
+    def _skip_this_question(self, current_question: "QuestionBase") -> bool:
+        """Determine if the current question should be skipped.
+
+        :param current_question: the question to be answered.
+        """
+        current_question_index = self.to_index[current_question.question_name]
+
+        answers = self.answers | self.scenario | self.agent["traits"]
+        skip = self.survey.rule_collection.skip_question_before_running(
+            current_question_index, answers
+        )
+        return skip
+
+    def _handle_exception(self, e, question_name: str, task=None):
+        exception_entry = InterviewExceptionEntry(e)
+        if task:
+            task.task_status = TaskStatus.FAILED
+        self.exceptions.add(question_name, exception_entry)
+
+    async def _attempt_to_answer_question(
+        self, invigilator: "InvigilatorBase", task: asyncio.Task
+    ) -> "AgentResponseDict":
+        """Attempt to answer the question, and handle exceptions.
+
+        :param invigilator: the invigilator that will answer the question.
+        :param task: the task that is being run.
+
+        """
+        try:
+            return await asyncio.wait_for(
+                invigilator.async_answer_question(), timeout=TIMEOUT
+            )
+        except asyncio.TimeoutError as e:
+            self._handle_exception(e, invigilator.question.question_name, task)
+            raise InterviewTimeoutError(f"Task timed out after {TIMEOUT} seconds.")
+        except Exception as e:
+            self._handle_exception(e, invigilator.question.question_name, task)
+            raise e
+
+    def _cancel_skipped_questions(self, current_question: QuestionBase) -> None:
+        """Cancel the tasks for questions that are skipped.
+
+        :param current_question: the question that was just answered.
+
+        It first determines the next question, given the current question and the current answers.
+        If the next question is the end of the survey, it cancels all remaining tasks.
+        If the next question is after the current question, it cancels all tasks between the current question and the next question.
+        """
+        current_question_index: int = self.to_index[current_question.question_name]
+
+        next_question: Union[
+            int, EndOfSurvey
+        ] = self.survey.rule_collection.next_question(
+            q_now=current_question_index,
+            answers=self.answers | self.scenario | self.agent["traits"],
+        )
+
+        next_question_index = next_question.next_q
+
+        def cancel_between(start, end):
+            """Cancel the tasks between the start and end indices."""
+            for i in range(start, end):
+                self.tasks[i].cancel()
+
+        if next_question_index == EndOfSurvey:
+            cancel_between(current_question_index + 1, len(self.survey.questions))
+            return
+
+        if next_question_index > (current_question_index + 1):
+            cancel_between(current_question_index + 1, next_question_index)

edsl/jobs/interviews/{InterviewExceptionCollection.py → interview_exception_tracking.py}

@@ -1,26 +1,74 @@
+import traceback
+import datetime
+import time
 from collections import UserDict
 
 from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 
+#                 #traceback=traceback.format_exc(),
+#                 #traceback = frame_summary_to_dict(traceback.extract_tb(e.__traceback__))
+#                 #traceback = [frame_summary_to_dict(f) for f in traceback.extract_tb(e.__traceback__)]
 
-class InterviewExceptionCollection(UserDict):
-    """A collection of exceptions that occurred during the interview."""
+# class InterviewExceptionEntry:
+#     """Class to record an exception that occurred during the interview.
+
+#     >>> entry = InterviewExceptionEntry.example()
+#     >>> entry.to_dict()['exception']
+#     "ValueError('An error occurred.')"
+#     """
+
+#     def __init__(self, exception: Exception):
+#         self.time = datetime.datetime.now().isoformat()
+#         self.exception = exception
+
+#     def __getitem__(self, key):
+#         # Support dict-like access obj['a']
+#         return str(getattr(self, key))
+
+#     @classmethod
+#     def example(cls):
+#         try:
+#             raise ValueError("An error occurred.")
+#         except Exception as e:
+#             entry = InterviewExceptionEntry(e)
+#         return entry
+
+#     @property
+#     def traceback(self):
+#         """Return the exception as HTML."""
+#         e = self.exception
+#         tb_str = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
+#         return tb_str
 
-    def __init__(self):
-        super().__init__()
-        self.fixed = set()
 
-    def unfixed_exceptions(self) -> list:
-        """Return a list of unfixed exceptions."""
-        return {k: v for k, v in self.data.items() if k not in self.fixed}
+#     @property
+#     def html(self):
+#         from rich.console import Console
+#         from rich.table import Table
+#         from rich.traceback import Traceback
 
-    def num_unfixed(self) -> list:
-        """Return a list of unfixed questions."""
-        return len([k for k in self.data.keys() if k not in self.fixed])
+#         from io import StringIO
+#         html_output = StringIO()
 
-    def record_fixed_question(self, question_name: str) -> None:
-        """Record that a question has been fixed."""
-        self.fixed.add(question_name)
+#         console = Console(file=html_output, record=True)
+#         tb = Traceback(show_locals=True)
+#         console.print(tb)
+
+#         tb = Traceback.from_exception(type(self.exception), self.exception, self.exception.__traceback__, show_locals=True)
+#         console.print(tb)
+#         return html_output.getvalue()
+
+#     def to_dict(self) -> dict:
+#         """Return the exception as a dictionary."""
+#         return {
+#             'exception': repr(self.exception),
+#             'time': self.time,
+#             'traceback': self.traceback
+#         }
+
+
+class InterviewExceptionCollection(UserDict):
+    """A collection of exceptions that occurred during the interview."""
 
     def add(self, question_name: str, entry: InterviewExceptionEntry) -> None:
         """Add an exception entry to the collection."""
@@ -32,6 +80,12 @@ class InterviewExceptionCollection(UserDict):
     def to_dict(self, include_traceback=True) -> dict:
         """Return the collection of exceptions as a dictionary."""
         newdata = {k: [e.to_dict() for e in v] for k, v in self.data.items()}
+        # if not include_traceback:
+        #     for question in newdata:
+        #         for exception in newdata[question]:
+        #             exception[
+        #                 "traceback"
+        #             ] = "Traceback removed. Set include_traceback=True to include."
         return newdata
 
     def _repr_html_(self) -> str:

edsl/jobs/interviews/retry_management.py

@@ -0,0 +1,37 @@
+from edsl import CONFIG
+
+from tenacity import (
+    retry,
+    wait_exponential,
+    stop_after_attempt,
+    retry_if_exception_type,
+    before_sleep,
+)
+
+EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
+EDSL_MAX_BACKOFF_SEC = float(CONFIG.get("EDSL_MAX_BACKOFF_SEC"))
+EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
+
+
+def print_retry(retry_state, print_to_terminal=True):
+    "Prints details on tenacity retries."
+    attempt_number = retry_state.attempt_number
+    exception = retry_state.outcome.exception()
+    wait_time = retry_state.next_action.sleep
+    if print_to_terminal:
+        print(
+            f"Attempt {attempt_number} failed with exception:" f"{exception}",
+            f"now waiting {wait_time:.2f} seconds before retrying."
+            f"Parameters: start={EDSL_BACKOFF_START_SEC}, max={EDSL_MAX_BACKOFF_SEC}, max_attempts={EDSL_MAX_ATTEMPTS}."
+            "\n\n",
+        )
+
+
+retry_strategy = retry(
+    wait=wait_exponential(
+        multiplier=EDSL_BACKOFF_START_SEC, max=EDSL_MAX_BACKOFF_SEC
+    ),  # Exponential back-off starting at 1s, doubling, maxing out at 60s
+    stop=stop_after_attempt(EDSL_MAX_ATTEMPTS),  # Stop after 5 attempts
+    # retry=retry_if_exception_type(Exception),  # Customize this as per your specific retry-able exception
+    before_sleep=print_retry,  # Use custom print function for retries
+)