edsl 0.1.33.dev2__py3-none-any.whl → 0.1.33.dev3__py3-none-any.whl

edsl/jobs/interviews/Interview.py

@@ -4,10 +4,18 @@ from __future__ import annotations
 import asyncio
 from typing import Any, Type, List, Generator, Optional, Union
 
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_exponential,
+    retry_if_exception_type,
+    RetryError,
+)
+
 from edsl import CONFIG
 from edsl.surveys.base import EndOfSurvey
 from edsl.exceptions import QuestionAnswerValidationError
-from edsl.exceptions import InterviewTimeoutError
+from edsl.exceptions import QuestionAnswerValidationError
 from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
 
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
@@ -15,21 +23,18 @@ from edsl.jobs.Answers import Answers
 from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
 from edsl.jobs.tasks.TaskCreators import TaskCreators
 from edsl.jobs.interviews.InterviewStatusLog import InterviewStatusLog
-from edsl.jobs.interviews.interview_exception_tracking import (
+from edsl.jobs.interviews.InterviewExceptionCollection import (
     InterviewExceptionCollection,
 )
-from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
-from edsl.jobs.interviews.retry_management import retry_strategy
+
 from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
 
 from edsl.surveys.base import EndOfSurvey
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
 from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
-from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.tasks.task_status_enum import TaskStatus
 from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
 
-from edsl.exceptions import QuestionAnswerValidationError
 
 from edsl import Agent, Survey, Scenario, Cache
 from edsl.language_models import LanguageModel
@@ -39,8 +44,11 @@ from edsl.agents.InvigilatorBase import InvigilatorBase
 from edsl.exceptions.language_models import LanguageModelNoResponseError
 
 
-class RetryableLanguageModelNoResponseError(LanguageModelNoResponseError):
-    pass
+from edsl import CONFIG
+
+EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
+EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
+EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
 
 
 class Interview(InterviewStatusMixin):
@@ -97,14 +105,17 @@ class Interview(InterviewStatusMixin):
         self.debug = debug
         self.iteration = iteration
         self.cache = cache
-        self.answers: dict[str, str] = (
-            Answers()
-        )  # will get filled in as interview progresses
+        self.answers: dict[
+            str, str
+        ] = Answers()  # will get filled in as interview progresses
         self.sidecar_model = sidecar_model
 
+        # self.stop_on_exception = False
+
         # Trackers
         self.task_creators = TaskCreators()  # tracks the task creators
         self.exceptions = InterviewExceptionCollection()
+
         self._task_status_log_dict = InterviewStatusLog()
         self.skip_retry = skip_retry
         self.raise_validation_errors = raise_validation_errors
@@ -257,44 +268,80 @@ class Interview(InterviewStatusMixin):
     ) -> "AgentResponseDict":
         """Answer a question and records the task."""
 
-        invigilator = self._get_invigilator(question)
+        had_language_model_no_response_error = False
 
-        if self._skip_this_question(question):
-            response = invigilator.get_failed_task_result(
-                failure_reason="Question skipped."
-            )
+        @retry(
+            stop=stop_after_attempt(EDSL_MAX_ATTEMPTS),
+            wait=wait_exponential(
+                multiplier=EDSL_BACKOFF_START_SEC, max=EDSL_BACKOFF_MAX_SEC
+            ),
+            retry=retry_if_exception_type(LanguageModelNoResponseError),
+            reraise=True,
+        )
+        async def attempt_answer():
+            nonlocal had_language_model_no_response_error
 
-        try:
-            response: EDSLResultObjectInput = await invigilator.async_answer_question()
-            if response.validated:
-                self.answers.add_answer(response=response, question=question)
-                self._cancel_skipped_questions(question)
-            else:
-                if (
-                    hasattr(response, "exception_occurred")
-                    and response.exception_occurred
-                ):
-                    raise response.exception_occurred
+            invigilator = self._get_invigilator(question)
 
-        except QuestionAnswerValidationError as e:
-            # there's a response, but it couldn't be validated
-            self._handle_exception(e, invigilator, task)
+            if self._skip_this_question(question):
+                return invigilator.get_failed_task_result(
+                    failure_reason="Question skipped."
+                )
 
-        except asyncio.TimeoutError as e:
-            # the API timed-out - this is recorded but as a response isn't generated, the LanguageModelNoResponseError will also be raised
-            self._handle_exception(e, invigilator, task)
+            try:
+                response: EDSLResultObjectInput = (
+                    await invigilator.async_answer_question()
+                )
+                if response.validated:
+                    self.answers.add_answer(response=response, question=question)
+                    self._cancel_skipped_questions(question)
+                else:
+                    if (
+                        hasattr(response, "exception_occurred")
+                        and response.exception_occurred
+                    ):
+                        raise response.exception_occurred
+
+            except QuestionAnswerValidationError as e:
+                self._handle_exception(e, invigilator, task)
+                return invigilator.get_failed_task_result(
+                    failure_reason="Question answer validation failed."
+                )
 
-        except Exception as e:
-            # there was some other exception
-            self._handle_exception(e, invigilator, task)
+            except asyncio.TimeoutError as e:
+                self._handle_exception(e, invigilator, task)
+                had_language_model_no_response_error = True
+                raise LanguageModelNoResponseError(
+                    f"Language model timed out for question '{question.question_name}.'"
+                )
 
-        if "response" not in locals():
+            except Exception as e:
+                self._handle_exception(e, invigilator, task)
 
-            raise LanguageModelNoResponseError(
-                f"Language model did not return a response for question '{question.question_name}.'"
-            )
+            if "response" not in locals():
+                had_language_model_no_response_error = True
+                raise LanguageModelNoResponseError(
+                    f"Language model did not return a response for question '{question.question_name}.'"
+                )
+
+            # if it gets here, it means the no response error was fixed
+            if (
+                question.question_name in self.exceptions
+                and had_language_model_no_response_error
+            ):
+                self.exceptions.record_fixed_question(question.question_name)
 
-        return response
+            return response
+
+        try:
+            return await attempt_answer()
+        except RetryError as retry_error:
+            # All retries have failed for LanguageModelNoResponseError
+            original_error = retry_error.last_attempt.exception()
+            self._handle_exception(
+                original_error, self._get_invigilator(question), task
+            )
+            raise original_error  # Re-raise the original error after handling
 
     def _get_invigilator(self, question: QuestionBase) -> InvigilatorBase:
         """Return an invigilator for the given question.
@@ -334,14 +381,32 @@ class Interview(InterviewStatusMixin):
     def _handle_exception(
         self, e: Exception, invigilator: "InvigilatorBase", task=None
     ):
+        import copy
+
+        # breakpoint()
+
+        answers = copy.copy(self.answers)
         exception_entry = InterviewExceptionEntry(
             exception=e,
             invigilator=invigilator,
+            answers=answers,
         )
         if task:
             task.task_status = TaskStatus.FAILED
         self.exceptions.add(invigilator.question.question_name, exception_entry)
 
+        if self.raise_validation_errors:
+            if isinstance(e, QuestionAnswerValidationError):
+                raise e
+
+        if hasattr(self, "stop_on_exception"):
+            stop_on_exception = self.stop_on_exception
+        else:
+            stop_on_exception = False
+
+        if stop_on_exception:
+            raise e
+
     def _cancel_skipped_questions(self, current_question: QuestionBase) -> None:
         """Cancel the tasks for questions that are skipped.
 
@@ -353,11 +418,11 @@ class Interview(InterviewStatusMixin):
         """
         current_question_index: int = self.to_index[current_question.question_name]
 
-        next_question: Union[int, EndOfSurvey] = (
-            self.survey.rule_collection.next_question(
-                q_now=current_question_index,
-                answers=self.answers | self.scenario | self.agent["traits"],
-            )
+        next_question: Union[
+            int, EndOfSurvey
+        ] = self.survey.rule_collection.next_question(
+            q_now=current_question_index,
+            answers=self.answers | self.scenario | self.agent["traits"],
         )
 
         next_question_index = next_question.next_q
@@ -411,6 +476,7 @@ class Interview(InterviewStatusMixin):
         asyncio.exceptions.CancelledError
         """
         self.sidecar_model = sidecar_model
+        self.stop_on_exception = stop_on_exception
 
         # if no model bucket is passed, create an 'infinity' bucket with no rate limits
         if model_buckets is None or hasattr(self.agent, "answer_question_directly"):
@@ -424,7 +490,9 @@ class Interview(InterviewStatusMixin):
         self.invigilators = [
             self._get_invigilator(question) for question in self.survey.questions
         ]
-        await asyncio.gather(*self.tasks, return_exceptions=not stop_on_exception)
+        await asyncio.gather(
+            *self.tasks, return_exceptions=not stop_on_exception
+        )  # not stop_on_exception)
         self.answers.replace_missing_answers_with_none(self.survey)
         valid_results = list(self._extract_valid_results())
         return self.answers, valid_results

edsl/jobs/interviews/{interview_exception_tracking.py → InterviewExceptionCollection.py}

@@ -6,6 +6,22 @@ from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 class InterviewExceptionCollection(UserDict):
     """A collection of exceptions that occurred during the interview."""
 
+    def __init__(self):
+        super().__init__()
+        self.fixed = set()
+
+    def unfixed_exceptions(self) -> list:
+        """Return a list of unfixed exceptions."""
+        return {k: v for k, v in self.data.items() if k not in self.fixed}
+
+    def num_unfixed(self) -> list:
+        """Return a list of unfixed questions."""
+        return len([k for k in self.data.keys() if k not in self.fixed])
+
+    def record_fixed_question(self, question_name: str) -> None:
+        """Record that a question has been fixed."""
+        self.fixed.add(question_name)
+
     def add(self, question_name: str, entry: InterviewExceptionEntry) -> None:
         """Add an exception entry to the collection."""
         question_name = question_name

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -15,12 +15,14 @@ class InterviewExceptionEntry:
         # failed_question: FailedQuestion,
         invigilator: "Invigilator",
         traceback_format="text",
+        answers=None,
     ):
         self.time = datetime.datetime.now().isoformat()
         self.exception = exception
         # self.failed_question = failed_question
         self.invigilator = invigilator
         self.traceback_format = traceback_format
+        self.answers = answers
 
     @property
     def question_type(self):

edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -3,40 +3,25 @@ import time
 import math
 import asyncio
 import functools
+import threading
 from typing import Coroutine, List, AsyncGenerator, Optional, Union, Generator
 from contextlib import contextmanager
 from collections import UserList
 
+from edsl.results.Results import Results
+from rich.live import Live
+from rich.console import Console
+
 from edsl import shared_globals
 from edsl.jobs.interviews.Interview import Interview
-from edsl.jobs.runners.JobsRunnerStatusMixin import JobsRunnerStatusMixin
+from edsl.jobs.runners.JobsRunnerStatus import JobsRunnerStatus
+
 from edsl.jobs.tasks.TaskHistory import TaskHistory
 from edsl.jobs.buckets.BucketCollection import BucketCollection
 from edsl.utilities.decorators import jupyter_nb_handler
 from edsl.data.Cache import Cache
 from edsl.results.Result import Result
 from edsl.results.Results import Results
-from edsl.jobs.FailedQuestion import FailedQuestion
-
-
-def cache_with_timeout(timeout):
-    """ "Used to keep the generate table from being run too frequetly."""
-
-    def decorator(func):
-        cached_result = {}
-        last_computation_time = [0]  # Using list to store mutable value
-
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            current_time = time.time()
-            if (current_time - last_computation_time[0]) >= timeout:
-                cached_result["value"] = func(*args, **kwargs)
-                last_computation_time[0] = current_time
-            return cached_result["value"]
-
-        return wrapper
-
-    return decorator
 
 
 class StatusTracker(UserList):
@@ -48,7 +33,7 @@ class StatusTracker(UserList):
         return print(f"Completed: {len(self.data)} of {self.total_tasks}", end="\r")
 
 
-class JobsRunnerAsyncio(JobsRunnerStatusMixin):
+class JobsRunnerAsyncio:
     """A class for running a collection of interviews asynchronously.
 
     It gets instaniated from a Jobs object.
@@ -57,11 +42,12 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
 
     def __init__(self, jobs: "Jobs"):
         self.jobs = jobs
-        # this creates the interviews, which can take a while
         self.interviews: List["Interview"] = jobs.interviews()
         self.bucket_collection: "BucketCollection" = jobs.bucket_collection
         self.total_interviews: List["Interview"] = []
 
+        # self.jobs_runner_status = JobsRunnerStatus(self, n=1)
+
     async def run_async_generator(
         self,
         cache: "Cache",
@@ -79,6 +65,7 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
         :param stop_on_exception: Whether to stop the interview if an exception is raised
         :param sidecar_model: a language model to use in addition to the interview's model
         :param total_interviews: A list of interviews to run can be provided instead.
+        :param raise_validation_errors: Whether to raise validation errors
         """
         tasks = []
         if total_interviews:  # was already passed in total interviews
@@ -88,8 +75,6 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
                 self._populate_total_interviews(n=n)
             )  # Populate self.total_interviews before creating tasks
 
-        # print("Interviews created")
-
         for interview in self.total_interviews:
             interviewing_task = self._build_interview_task(
                 interview=interview,
@@ -99,11 +84,9 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
             )
             tasks.append(asyncio.create_task(interviewing_task))
 
-        # print("Tasks created")
-
         for task in asyncio.as_completed(tasks):
-            # print(f"Task {task} completed")
             result = await task
+            self.jobs_runner_status.add_completed_interview(result)
             yield result
 
     def _populate_total_interviews(
@@ -122,6 +105,8 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
                     yield interview
 
     async def run_async(self, cache: Optional["Cache"] = None, n: int = 1) -> Results:
+        """Used for some other modules that have a non-standard way of running interviews."""
+        self.jobs_runner_status = JobsRunnerStatus(self, n=n)
         self.cache = Cache() if cache is None else cache
         data = []
         async for result in self.run_async_generator(cache=self.cache, n=n):
@@ -157,12 +142,6 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
             raise_validation_errors=raise_validation_errors,
         )
 
-        # answer_key_names = {
-        #     k
-        #     for k in set(answer.keys())
-        #     if not k.endswith("_comment") and not k.endswith("_generated_tokens")
-        # }
-
         question_results = {}
         for result in valid_results:
             question_results[result.question_name] = result
@@ -174,24 +153,13 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
             for k in answer_key_names
         }
         comments_dict = {
-            "k" + "_comment": question_results[k].comment for k in answer_key_names
+            k + "_comment": question_results[k].comment for k in answer_key_names
         }
 
         # we should have a valid result for each question
         answer_dict = {k: answer[k] for k in answer_key_names}
         assert len(valid_results) == len(answer_key_names)
 
-        # breakpoint()
-        # generated_tokens_dict = {
-        #     k + "_generated_tokens": v.generated_tokens
-        #     for k, v in zip(answer_key_names, valid_results)
-        # }
-
-        # comments_dict = {
-        #    k + "_comment": v.comment for k, v in zip(answer_key_names, valid_results)
-        # }
-        # breakpoint()
-
         # TODO: move this down into Interview
         question_name_to_prompts = dict({})
         for result in valid_results:
@@ -226,7 +194,6 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
             )
             raw_model_results_dictionary[question_name + "_one_usd_buys"] = one_use_buys
 
-        # breakpoint()
         result = Result(
             agent=interview.agent,
             scenario=interview.scenario,
@@ -247,6 +214,62 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
     def elapsed_time(self):
         return time.monotonic() - self.start_time
 
+    def process_results(
+        self, raw_results: Results, cache: Cache, print_exceptions: bool
+    ):
+        interview_lookup = {
+            hash(interview): index
+            for index, interview in enumerate(self.total_interviews)
+        }
+        interview_hashes = list(interview_lookup.keys())
+
+        results = Results(
+            survey=self.jobs.survey,
+            data=sorted(
+                raw_results, key=lambda x: interview_hashes.index(x.interview_hash)
+            ),
+        )
+        results.cache = cache
+        results.task_history = TaskHistory(
+            self.total_interviews, include_traceback=False
+        )
+        results.has_unfixed_exceptions = results.task_history.has_unfixed_exceptions
+        results.bucket_collection = self.bucket_collection
+
+        if results.has_unfixed_exceptions and print_exceptions:
+            from edsl.scenarios.FileStore import HTMLFileStore
+            from edsl.config import CONFIG
+            from edsl.coop.coop import Coop
+
+            msg = f"Exceptions were raised in {len(results.task_history.indices)} out of {len(self.total_interviews)} interviews.\n"
+
+            if len(results.task_history.indices) > 5:
+                msg += f"Exceptions were raised in the following interviews: {results.task_history.indices}.\n"
+
+            print(msg)
+            # this is where exceptions are opening up
+            filepath = results.task_history.html(
+                cta="Open report to see details.",
+                open_in_browser=True,
+                return_link=True,
+            )
+
+            try:
+                coop = Coop()
+                user_edsl_settings = coop.edsl_settings
+                remote_logging = user_edsl_settings["remote_logging"]
+            except Exception as e:
+                print(e)
+                remote_logging = False
+            if remote_logging:
+                filestore = HTMLFileStore(filepath)
+                coop_details = filestore.push(description="Error report")
+                print(coop_details)
+
+            print("Also see: https://docs.expectedparrot.com/en/latest/exceptions.html")
+
+        return results
+
     @jupyter_nb_handler
     async def run(
         self,
@@ -259,24 +282,16 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
         raise_validation_errors: bool = False,
     ) -> "Coroutine":
         """Runs a collection of interviews, handling both async and sync contexts."""
-        from rich.console import Console
 
-        console = Console()
         self.results = []
         self.start_time = time.monotonic()
         self.completed = False
         self.cache = cache
         self.sidecar_model = sidecar_model
 
-        from edsl.results.Results import Results
-        from rich.live import Live
-        from rich.console import Console
-
-        @cache_with_timeout(1)
-        def generate_table():
-            return self.status_table(self.results, self.elapsed_time)
+        self.jobs_runner_status = JobsRunnerStatus(self, n=n)
 
-        async def process_results(cache, progress_bar_context=None):
+        async def process_results(cache):
             """Processes results from interviews."""
             async for result in self.run_async_generator(
                 n=n,
@@ -286,112 +301,22 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
                 raise_validation_errors=raise_validation_errors,
             ):
                 self.results.append(result)
-                if progress_bar_context:
-                    progress_bar_context.update(generate_table())
             self.completed = True
 
-        async def update_progress_bar(progress_bar_context):
-            """Updates the progress bar at fixed intervals."""
-            if progress_bar_context is None:
-                return
-
-            while True:
-                progress_bar_context.update(generate_table())
-                await asyncio.sleep(0.1)  # Update interval
-                if self.completed:
-                    break
-
-        @contextmanager
-        def conditional_context(condition, context_manager):
-            if condition:
-                with context_manager as cm:
-                    yield cm
-            else:
-                yield
-
-        with conditional_context(
-            progress_bar, Live(generate_table(), console=console, refresh_per_second=1)
-        ) as progress_bar_context:
-            with cache as c:
-                progress_task = asyncio.create_task(
-                    update_progress_bar(progress_bar_context)
-                )
-
-                try:
-                    await asyncio.gather(
-                        progress_task,
-                        process_results(
-                            cache=c, progress_bar_context=progress_bar_context
-                        ),
-                    )
-                except asyncio.CancelledError:
-                    pass
-                finally:
-                    progress_task.cancel()  # Cancel the progress_task when process_results is done
-                    await progress_task
-
-                    await asyncio.sleep(1)  # short delay to show the final status
-
-                    if progress_bar_context:
-                        progress_bar_context.update(generate_table())
-
-        # puts results in the same order as the total interviews
-        interview_lookup = {
-            hash(interview): index
-            for index, interview in enumerate(self.total_interviews)
-        }
-        interview_hashes = list(interview_lookup.keys())
-        self.results = sorted(
-            self.results, key=lambda x: interview_hashes.index(x.interview_hash)
-        )
-
-        results = Results(survey=self.jobs.survey, data=self.results)
-        task_history = TaskHistory(self.total_interviews, include_traceback=False)
-        results.task_history = task_history
-
-        results.failed_questions = {}
-        results.has_exceptions = task_history.has_exceptions
-
-        # breakpoint()
-        results.bucket_collection = self.bucket_collection
-
-        if results.has_exceptions:
-            # put the failed interviews in the results object as a list
-            failed_interviews = [
-                interview.duplicate(
-                    iteration=interview.iteration, cache=interview.cache
-                )
-                for interview in self.total_interviews
-                if interview.has_exceptions
-            ]
-
-            failed_questions = {}
-            for interview in self.total_interviews:
-                if interview.has_exceptions:
-                    index = interview_lookup[hash(interview)]
-                    failed_questions[index] = interview.failed_questions
+        def run_progress_bar():
+            """Runs the progress bar in a separate thread."""
+            self.jobs_runner_status.update_progress()
 
-            results.failed_questions = failed_questions
+        if progress_bar:
+            progress_thread = threading.Thread(target=run_progress_bar)
+            progress_thread.start()
 
-            from edsl.jobs.Jobs import Jobs
+        with cache as c:
+            await process_results(cache=c)
 
-            results.failed_jobs = Jobs.from_interviews(
-                [interview for interview in failed_interviews]
-            )
-            if print_exceptions:
-                msg = f"Exceptions were raised in {len(results.task_history.indices)} out of {len(self.total_interviews)} interviews.\n"
-
-                if len(results.task_history.indices) > 5:
-                    msg += f"Exceptions were raised in the following interviews: {results.task_history.indices}.\n"
-
-                shared_globals["edsl_runner_exceptions"] = task_history
-                print(msg)
-                # this is where exceptions are opening up
-                task_history.html(
-                    cta="Open report to see details.", open_in_browser=True
-                )
-                print(
-                    "Also see: https://docs.expectedparrot.com/en/latest/exceptions.html"
-                )
+        if progress_bar:
+            progress_thread.join()
 
-        return results
+        return self.process_results(
+            raw_results=self.results, cache=cache, print_exceptions=print_exceptions
+        )