edsl 0.1.33.dev2__py3-none-any.whl → 0.1.34__py3-none-any.whl

edsl/jobs/interviews/Interview.py

@@ -3,11 +3,20 @@
 from __future__ import annotations
 import asyncio
 from typing import Any, Type, List, Generator, Optional, Union
+import copy
+
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_exponential,
+    retry_if_exception_type,
+    RetryError,
+)
 
 from edsl import CONFIG
 from edsl.surveys.base import EndOfSurvey
 from edsl.exceptions import QuestionAnswerValidationError
-from edsl.exceptions import InterviewTimeoutError
+from edsl.exceptions import QuestionAnswerValidationError
 from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
 
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
@@ -15,21 +24,18 @@ from edsl.jobs.Answers import Answers
 from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
 from edsl.jobs.tasks.TaskCreators import TaskCreators
 from edsl.jobs.interviews.InterviewStatusLog import InterviewStatusLog
-from edsl.jobs.interviews.interview_exception_tracking import (
+from edsl.jobs.interviews.InterviewExceptionCollection import (
     InterviewExceptionCollection,
 )
-from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
-from edsl.jobs.interviews.retry_management import retry_strategy
+
 from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
 
 from edsl.surveys.base import EndOfSurvey
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
 from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
-from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.tasks.task_status_enum import TaskStatus
 from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
 
-from edsl.exceptions import QuestionAnswerValidationError
 
 from edsl import Agent, Survey, Scenario, Cache
 from edsl.language_models import LanguageModel
@@ -39,8 +45,11 @@ from edsl.agents.InvigilatorBase import InvigilatorBase
 from edsl.exceptions.language_models import LanguageModelNoResponseError
 
 
-class RetryableLanguageModelNoResponseError(LanguageModelNoResponseError):
-    pass
+from edsl import CONFIG
+
+EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
+EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
+EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
 
 
 class Interview(InterviewStatusMixin):
@@ -91,20 +100,25 @@ class Interview(InterviewStatusMixin):
 
         """
         self.agent = agent
-        self.survey = survey
+        # what I would like to do
+        self.survey = copy.deepcopy(survey)  # survey  copy.deepcopy(survey)
+        # self.survey = survey
         self.scenario = scenario
         self.model = model
         self.debug = debug
         self.iteration = iteration
         self.cache = cache
-        self.answers: dict[str, str] = (
-            Answers()
-        )  # will get filled in as interview progresses
+        self.answers: dict[
+            str, str
+        ] = Answers()  # will get filled in as interview progresses
         self.sidecar_model = sidecar_model
 
+        # self.stop_on_exception = False
+
         # Trackers
         self.task_creators = TaskCreators()  # tracks the task creators
         self.exceptions = InterviewExceptionCollection()
+
         self._task_status_log_dict = InterviewStatusLog()
         self.skip_retry = skip_retry
         self.raise_validation_errors = raise_validation_errors
@@ -237,17 +251,24 @@ class Interview(InterviewStatusMixin):
 
     def _get_estimated_request_tokens(self, question) -> float:
         """Estimate the number of tokens that will be required to run the focal task."""
+        from edsl.scenarios.FileStore import FileStore
+
         invigilator = self._get_invigilator(question=question)
         # TODO: There should be a way to get a more accurate estimate.
         combined_text = ""
+        file_tokens = 0
         for prompt in invigilator.get_prompts().values():
             if hasattr(prompt, "text"):
                 combined_text += prompt.text
             elif isinstance(prompt, str):
                 combined_text += prompt
+            elif isinstance(prompt, list):
+                for file in prompt:
+                    if isinstance(file, FileStore):
+                        file_tokens += file.size * 0.25
             else:
                 raise ValueError(f"Prompt is of type {type(prompt)}")
-        return len(combined_text) / 4.0
+        return len(combined_text) / 4.0 + file_tokens
 
     async def _answer_question_and_record_task(
         self,
@@ -257,44 +278,83 @@ class Interview(InterviewStatusMixin):
     ) -> "AgentResponseDict":
         """Answer a question and records the task."""
 
-        invigilator = self._get_invigilator(question)
+        had_language_model_no_response_error = False
 
-        if self._skip_this_question(question):
-            response = invigilator.get_failed_task_result(
-                failure_reason="Question skipped."
-            )
+        @retry(
+            stop=stop_after_attempt(EDSL_MAX_ATTEMPTS),
+            wait=wait_exponential(
+                multiplier=EDSL_BACKOFF_START_SEC, max=EDSL_BACKOFF_MAX_SEC
+            ),
+            retry=retry_if_exception_type(LanguageModelNoResponseError),
+            reraise=True,
+        )
+        async def attempt_answer():
+            nonlocal had_language_model_no_response_error
 
-        try:
-            response: EDSLResultObjectInput = await invigilator.async_answer_question()
-            if response.validated:
-                self.answers.add_answer(response=response, question=question)
-                self._cancel_skipped_questions(question)
-            else:
-                if (
-                    hasattr(response, "exception_occurred")
-                    and response.exception_occurred
-                ):
-                    raise response.exception_occurred
+            invigilator = self._get_invigilator(question)
 
-        except QuestionAnswerValidationError as e:
-            # there's a response, but it couldn't be validated
-            self._handle_exception(e, invigilator, task)
+            if self._skip_this_question(question):
+                return invigilator.get_failed_task_result(
+                    failure_reason="Question skipped."
+                )
 
-        except asyncio.TimeoutError as e:
-            # the API timed-out - this is recorded but as a response isn't generated, the LanguageModelNoResponseError will also be raised
-            self._handle_exception(e, invigilator, task)
+            try:
+                response: EDSLResultObjectInput = (
+                    await invigilator.async_answer_question()
+                )
+                if response.validated:
+                    self.answers.add_answer(response=response, question=question)
+                    self._cancel_skipped_questions(question)
+                else:
+                    # When a question is not validated, it is not added to the answers.
+                    # this should also cancel and dependent children questions.
+                    # Is that happening now?
+                    if (
+                        hasattr(response, "exception_occurred")
+                        and response.exception_occurred
+                    ):
+                        raise response.exception_occurred
+
+            except QuestionAnswerValidationError as e:
+                self._handle_exception(e, invigilator, task)
+                return invigilator.get_failed_task_result(
+                    failure_reason="Question answer validation failed."
+                )
 
-        except Exception as e:
-            # there was some other exception
-            self._handle_exception(e, invigilator, task)
+            except asyncio.TimeoutError as e:
+                self._handle_exception(e, invigilator, task)
+                had_language_model_no_response_error = True
+                raise LanguageModelNoResponseError(
+                    f"Language model timed out for question '{question.question_name}.'"
+                )
 
-        if "response" not in locals():
+            except Exception as e:
+                self._handle_exception(e, invigilator, task)
 
-            raise LanguageModelNoResponseError(
-                f"Language model did not return a response for question '{question.question_name}.'"
-            )
+            if "response" not in locals():
+                had_language_model_no_response_error = True
+                raise LanguageModelNoResponseError(
+                    f"Language model did not return a response for question '{question.question_name}.'"
+                )
+
+            # if it gets here, it means the no response error was fixed
+            if (
+                question.question_name in self.exceptions
+                and had_language_model_no_response_error
+            ):
+                self.exceptions.record_fixed_question(question.question_name)
 
-        return response
+            return response
+
+        try:
+            return await attempt_answer()
+        except RetryError as retry_error:
+            # All retries have failed for LanguageModelNoResponseError
+            original_error = retry_error.last_attempt.exception()
+            self._handle_exception(
+                original_error, self._get_invigilator(question), task
+            )
+            raise original_error  # Re-raise the original error after handling
 
     def _get_invigilator(self, question: QuestionBase) -> InvigilatorBase:
         """Return an invigilator for the given question.
@@ -334,14 +394,32 @@ class Interview(InterviewStatusMixin):
     def _handle_exception(
         self, e: Exception, invigilator: "InvigilatorBase", task=None
     ):
+        import copy
+
+        # breakpoint()
+
+        answers = copy.copy(self.answers)
         exception_entry = InterviewExceptionEntry(
             exception=e,
             invigilator=invigilator,
+            answers=answers,
         )
         if task:
             task.task_status = TaskStatus.FAILED
         self.exceptions.add(invigilator.question.question_name, exception_entry)
 
+        if self.raise_validation_errors:
+            if isinstance(e, QuestionAnswerValidationError):
+                raise e
+
+        if hasattr(self, "stop_on_exception"):
+            stop_on_exception = self.stop_on_exception
+        else:
+            stop_on_exception = False
+
+        if stop_on_exception:
+            raise e
+
     def _cancel_skipped_questions(self, current_question: QuestionBase) -> None:
         """Cancel the tasks for questions that are skipped.
 
@@ -353,11 +431,11 @@ class Interview(InterviewStatusMixin):
         """
         current_question_index: int = self.to_index[current_question.question_name]
 
-        next_question: Union[int, EndOfSurvey] = (
-            self.survey.rule_collection.next_question(
-                q_now=current_question_index,
-                answers=self.answers | self.scenario | self.agent["traits"],
-            )
+        next_question: Union[
+            int, EndOfSurvey
+        ] = self.survey.rule_collection.next_question(
+            q_now=current_question_index,
+            answers=self.answers | self.scenario | self.agent["traits"],
         )
 
         next_question_index = next_question.next_q
@@ -411,6 +489,7 @@ class Interview(InterviewStatusMixin):
         asyncio.exceptions.CancelledError
         """
         self.sidecar_model = sidecar_model
+        self.stop_on_exception = stop_on_exception
 
         # if no model bucket is passed, create an 'infinity' bucket with no rate limits
         if model_buckets is None or hasattr(self.agent, "answer_question_directly"):
@@ -424,7 +503,9 @@ class Interview(InterviewStatusMixin):
         self.invigilators = [
             self._get_invigilator(question) for question in self.survey.questions
         ]
-        await asyncio.gather(*self.tasks, return_exceptions=not stop_on_exception)
+        await asyncio.gather(
+            *self.tasks, return_exceptions=not stop_on_exception
+        )  # not stop_on_exception)
         self.answers.replace_missing_answers_with_none(self.survey)
         valid_results = list(self._extract_valid_results())
         return self.answers, valid_results

edsl/jobs/interviews/{interview_exception_tracking.py → InterviewExceptionCollection.py}

@@ -6,6 +6,22 @@ from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 class InterviewExceptionCollection(UserDict):
     """A collection of exceptions that occurred during the interview."""
 
+    def __init__(self):
+        super().__init__()
+        self.fixed = set()
+
+    def unfixed_exceptions(self) -> list:
+        """Return a list of unfixed exceptions."""
+        return {k: v for k, v in self.data.items() if k not in self.fixed}
+
+    def num_unfixed(self) -> list:
+        """Return a list of unfixed questions."""
+        return len([k for k in self.data.keys() if k not in self.fixed])
+
+    def record_fixed_question(self, question_name: str) -> None:
+        """Record that a question has been fixed."""
+        self.fixed.add(question_name)
+
     def add(self, question_name: str, entry: InterviewExceptionEntry) -> None:
         """Add an exception entry to the collection."""
         question_name = question_name

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -15,12 +15,14 @@ class InterviewExceptionEntry:
         # failed_question: FailedQuestion,
         invigilator: "Invigilator",
         traceback_format="text",
+        answers=None,
     ):
         self.time = datetime.datetime.now().isoformat()
         self.exception = exception
         # self.failed_question = failed_question
         self.invigilator = invigilator
         self.traceback_format = traceback_format
+        self.answers = answers
 
     @property
     def question_type(self):