edsl 0.1.33__py3-none-any.whl → 0.1.33.dev2__py3-none-any.whl

edsl/jobs/interviews/Interview.py

@@ -4,18 +4,10 @@ from __future__ import annotations
 import asyncio
 from typing import Any, Type, List, Generator, Optional, Union
 
-from tenacity import (
-    retry,
-    stop_after_attempt,
-    wait_exponential,
-    retry_if_exception_type,
-    RetryError,
-)
-
 from edsl import CONFIG
 from edsl.surveys.base import EndOfSurvey
 from edsl.exceptions import QuestionAnswerValidationError
-from edsl.exceptions import QuestionAnswerValidationError
+from edsl.exceptions import InterviewTimeoutError
 from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
 
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
@@ -23,18 +15,21 @@ from edsl.jobs.Answers import Answers
 from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
 from edsl.jobs.tasks.TaskCreators import TaskCreators
 from edsl.jobs.interviews.InterviewStatusLog import InterviewStatusLog
-from edsl.jobs.interviews.InterviewExceptionCollection import (
+from edsl.jobs.interviews.interview_exception_tracking import (
     InterviewExceptionCollection,
 )
-
+from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
+from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
 
 from edsl.surveys.base import EndOfSurvey
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
 from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
+from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.tasks.task_status_enum import TaskStatus
 from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
 
+from edsl.exceptions import QuestionAnswerValidationError
 
 from edsl import Agent, Survey, Scenario, Cache
 from edsl.language_models import LanguageModel
@@ -44,11 +39,8 @@ from edsl.agents.InvigilatorBase import InvigilatorBase
 from edsl.exceptions.language_models import LanguageModelNoResponseError
 
 
-from edsl import CONFIG
-
-EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
-EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
-EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
+class RetryableLanguageModelNoResponseError(LanguageModelNoResponseError):
+    pass
 
 
 class Interview(InterviewStatusMixin):
@@ -105,17 +97,14 @@ class Interview(InterviewStatusMixin):
         self.debug = debug
         self.iteration = iteration
         self.cache = cache
-        self.answers: dict[
-            str, str
-        ] = Answers()  # will get filled in as interview progresses
+        self.answers: dict[str, str] = (
+            Answers()
+        )  # will get filled in as interview progresses
         self.sidecar_model = sidecar_model
 
-        # self.stop_on_exception = False
-
         # Trackers
         self.task_creators = TaskCreators()  # tracks the task creators
         self.exceptions = InterviewExceptionCollection()
-
         self._task_status_log_dict = InterviewStatusLog()
         self.skip_retry = skip_retry
         self.raise_validation_errors = raise_validation_errors
@@ -268,80 +257,44 @@ class Interview(InterviewStatusMixin):
     ) -> "AgentResponseDict":
         """Answer a question and records the task."""
 
-        had_language_model_no_response_error = False
-
-        @retry(
-            stop=stop_after_attempt(EDSL_MAX_ATTEMPTS),
-            wait=wait_exponential(
-                multiplier=EDSL_BACKOFF_START_SEC, max=EDSL_BACKOFF_MAX_SEC
-            ),
-            retry=retry_if_exception_type(LanguageModelNoResponseError),
-            reraise=True,
-        )
-        async def attempt_answer():
-            nonlocal had_language_model_no_response_error
-
-            invigilator = self._get_invigilator(question)
-
-            if self._skip_this_question(question):
-                return invigilator.get_failed_task_result(
-                    failure_reason="Question skipped."
-                )
+        invigilator = self._get_invigilator(question)
 
-            try:
-                response: EDSLResultObjectInput = (
-                    await invigilator.async_answer_question()
-                )
-                if response.validated:
-                    self.answers.add_answer(response=response, question=question)
-                    self._cancel_skipped_questions(question)
-                else:
-                    if (
-                        hasattr(response, "exception_occurred")
-                        and response.exception_occurred
-                    ):
-                        raise response.exception_occurred
-
-            except QuestionAnswerValidationError as e:
-                self._handle_exception(e, invigilator, task)
-                return invigilator.get_failed_task_result(
-                    failure_reason="Question answer validation failed."
-                )
+        if self._skip_this_question(question):
+            response = invigilator.get_failed_task_result(
+                failure_reason="Question skipped."
+            )
 
-            except asyncio.TimeoutError as e:
-                self._handle_exception(e, invigilator, task)
-                had_language_model_no_response_error = True
-                raise LanguageModelNoResponseError(
-                    f"Language model timed out for question '{question.question_name}.'"
-                )
+        try:
+            response: EDSLResultObjectInput = await invigilator.async_answer_question()
+            if response.validated:
+                self.answers.add_answer(response=response, question=question)
+                self._cancel_skipped_questions(question)
+            else:
+                if (
+                    hasattr(response, "exception_occurred")
+                    and response.exception_occurred
+                ):
+                    raise response.exception_occurred
 
-            except Exception as e:
-                self._handle_exception(e, invigilator, task)
+        except QuestionAnswerValidationError as e:
+            # there's a response, but it couldn't be validated
+            self._handle_exception(e, invigilator, task)
 
-            if "response" not in locals():
-                had_language_model_no_response_error = True
-                raise LanguageModelNoResponseError(
-                    f"Language model did not return a response for question '{question.question_name}.'"
-                )
+        except asyncio.TimeoutError as e:
+            # the API timed-out - this is recorded but as a response isn't generated, the LanguageModelNoResponseError will also be raised
+            self._handle_exception(e, invigilator, task)
 
-            # if it gets here, it means the no response error was fixed
-            if (
-                question.question_name in self.exceptions
-                and had_language_model_no_response_error
-            ):
-                self.exceptions.record_fixed_question(question.question_name)
+        except Exception as e:
+            # there was some other exception
+            self._handle_exception(e, invigilator, task)
 
-            return response
+        if "response" not in locals():
 
-        try:
-            return await attempt_answer()
-        except RetryError as retry_error:
-            # All retries have failed for LanguageModelNoResponseError
-            original_error = retry_error.last_attempt.exception()
-            self._handle_exception(
-                original_error, self._get_invigilator(question), task
+            raise LanguageModelNoResponseError(
+                f"Language model did not return a response for question '{question.question_name}.'"
             )
-            raise original_error  # Re-raise the original error after handling
+
+        return response
 
     def _get_invigilator(self, question: QuestionBase) -> InvigilatorBase:
         """Return an invigilator for the given question.
@@ -381,32 +334,14 @@ class Interview(InterviewStatusMixin):
     def _handle_exception(
         self, e: Exception, invigilator: "InvigilatorBase", task=None
     ):
-        import copy
-
-        # breakpoint()
-
-        answers = copy.copy(self.answers)
         exception_entry = InterviewExceptionEntry(
             exception=e,
             invigilator=invigilator,
-            answers=answers,
         )
         if task:
             task.task_status = TaskStatus.FAILED
         self.exceptions.add(invigilator.question.question_name, exception_entry)
 
-        if self.raise_validation_errors:
-            if isinstance(e, QuestionAnswerValidationError):
-                raise e
-
-        if hasattr(self, "stop_on_exception"):
-            stop_on_exception = self.stop_on_exception
-        else:
-            stop_on_exception = False
-
-        if stop_on_exception:
-            raise e
-
     def _cancel_skipped_questions(self, current_question: QuestionBase) -> None:
         """Cancel the tasks for questions that are skipped.
 
@@ -418,11 +353,11 @@ class Interview(InterviewStatusMixin):
         """
         current_question_index: int = self.to_index[current_question.question_name]
 
-        next_question: Union[
-            int, EndOfSurvey
-        ] = self.survey.rule_collection.next_question(
-            q_now=current_question_index,
-            answers=self.answers | self.scenario | self.agent["traits"],
+        next_question: Union[int, EndOfSurvey] = (
+            self.survey.rule_collection.next_question(
+                q_now=current_question_index,
+                answers=self.answers | self.scenario | self.agent["traits"],
+            )
         )
 
         next_question_index = next_question.next_q
@@ -476,7 +411,6 @@ class Interview(InterviewStatusMixin):
         asyncio.exceptions.CancelledError
         """
         self.sidecar_model = sidecar_model
-        self.stop_on_exception = stop_on_exception
 
         # if no model bucket is passed, create an 'infinity' bucket with no rate limits
         if model_buckets is None or hasattr(self.agent, "answer_question_directly"):
@@ -490,9 +424,7 @@ class Interview(InterviewStatusMixin):
         self.invigilators = [
             self._get_invigilator(question) for question in self.survey.questions
         ]
-        await asyncio.gather(
-            *self.tasks, return_exceptions=not stop_on_exception
-        )  # not stop_on_exception)
+        await asyncio.gather(*self.tasks, return_exceptions=not stop_on_exception)
         self.answers.replace_missing_answers_with_none(self.survey)
         valid_results = list(self._extract_valid_results())
         return self.answers, valid_results

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -15,14 +15,12 @@ class InterviewExceptionEntry:
         # failed_question: FailedQuestion,
         invigilator: "Invigilator",
         traceback_format="text",
-        answers=None,
     ):
         self.time = datetime.datetime.now().isoformat()
         self.exception = exception
         # self.failed_question = failed_question
         self.invigilator = invigilator
         self.traceback_format = traceback_format
-        self.answers = answers
 
     @property
     def question_type(self):

edsl/jobs/interviews/{InterviewExceptionCollection.py → interview_exception_tracking.py}

@@ -6,22 +6,6 @@ from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 class InterviewExceptionCollection(UserDict):
     """A collection of exceptions that occurred during the interview."""
 
-    def __init__(self):
-        super().__init__()
-        self.fixed = set()
-
-    def unfixed_exceptions(self) -> list:
-        """Return a list of unfixed exceptions."""
-        return {k: v for k, v in self.data.items() if k not in self.fixed}
-
-    def num_unfixed(self) -> list:
-        """Return a list of unfixed questions."""
-        return len([k for k in self.data.keys() if k not in self.fixed])
-
-    def record_fixed_question(self, question_name: str) -> None:
-        """Record that a question has been fixed."""
-        self.fixed.add(question_name)
-
     def add(self, question_name: str, entry: InterviewExceptionEntry) -> None:
         """Add an exception entry to the collection."""
         question_name = question_name

edsl/jobs/interviews/retry_management.py

@@ -0,0 +1,39 @@
+from edsl import CONFIG
+
+from tenacity import (
+    retry,
+    wait_exponential,
+    stop_after_attempt,
+    retry_if_exception_type,
+    before_sleep,
+)
+
+EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
+EDSL_MAX_BACKOFF_SEC = float(CONFIG.get("EDSL_MAX_BACKOFF_SEC"))
+EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
+
+
+def print_retry(retry_state, print_to_terminal=True):
+    "Prints details on tenacity retries."
+    attempt_number = retry_state.attempt_number
+    exception = retry_state.outcome.exception()
+    wait_time = retry_state.next_action.sleep
+    exception_name = type(exception).__name__
+    if print_to_terminal:
+        print(
+            f"Attempt {attempt_number} failed with exception '{exception_name}':"
+            f"{exception}",
+            f"now waiting {wait_time:.2f} seconds before retrying."
+            f"Parameters: start={EDSL_BACKOFF_START_SEC}, max={EDSL_MAX_BACKOFF_SEC}, max_attempts={EDSL_MAX_ATTEMPTS}."
+            "\n\n",
+        )
+
+
+retry_strategy = retry(
+    wait=wait_exponential(
+        multiplier=EDSL_BACKOFF_START_SEC, max=EDSL_MAX_BACKOFF_SEC
+    ),  # Exponential back-off starting at 1s, doubling, maxing out at 60s
+    stop=stop_after_attempt(EDSL_MAX_ATTEMPTS),  # Stop after 5 attempts
+    # retry=retry_if_exception_type(Exception),  # Customize this as per your specific retry-able exception
+    before_sleep=print_retry,  # Use custom print function for retries
+)