edsl 0.1.33__py3-none-any.whl → 0.1.33.dev1__py3-none-any.whl

edsl/jobs/interviews/Interview.py

@@ -1,76 +1,50 @@
 """This module contains the Interview class, which is responsible for conducting an interview asynchronously."""
 
 from __future__ import annotations
+import traceback
 import asyncio
-from typing import Any, Type, List, Generator, Optional, Union
-
-from tenacity import (
-    retry,
-    stop_after_attempt,
-    wait_exponential,
-    retry_if_exception_type,
-    RetryError,
-)
+import time
+from typing import Any, Type, List, Generator, Optional
 
-from edsl import CONFIG
+from edsl.jobs.Answers import Answers
 from edsl.surveys.base import EndOfSurvey
-from edsl.exceptions import QuestionAnswerValidationError
-from edsl.exceptions import QuestionAnswerValidationError
-from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
-
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
-from edsl.jobs.Answers import Answers
-from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
 from edsl.jobs.tasks.TaskCreators import TaskCreators
+
 from edsl.jobs.interviews.InterviewStatusLog import InterviewStatusLog
-from edsl.jobs.interviews.InterviewExceptionCollection import (
+from edsl.jobs.interviews.interview_exception_tracking import (
     InterviewExceptionCollection,
 )
-
-from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
-
-from edsl.surveys.base import EndOfSurvey
-from edsl.jobs.buckets.ModelBuckets import ModelBuckets
 from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
-from edsl.jobs.tasks.task_status_enum import TaskStatus
-from edsl.jobs.tasks.QuestionTaskCreator import QuestionTaskCreator
-
-
-from edsl import Agent, Survey, Scenario, Cache
-from edsl.language_models import LanguageModel
-from edsl.questions import QuestionBase
-from edsl.agents.InvigilatorBase import InvigilatorBase
-
-from edsl.exceptions.language_models import LanguageModelNoResponseError
+from edsl.jobs.interviews.retry_management import retry_strategy
+from edsl.jobs.interviews.InterviewTaskBuildingMixin import InterviewTaskBuildingMixin
+from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
 
+import asyncio
 
-from edsl import CONFIG
 
-EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
-EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
-EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
+def run_async(coro):
+    return asyncio.run(coro)
 
 
-class Interview(InterviewStatusMixin):
+class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
     """
     An 'interview' is one agent answering one survey, with one language model, for a given scenario.
 
     The main method is `async_conduct_interview`, which conducts the interview asynchronously.
-    Most of the class is dedicated to creating the tasks for each question in the survey, and then running them.
     """
 
     def __init__(
         self,
-        agent: Agent,
-        survey: Survey,
-        scenario: Scenario,
+        agent: "Agent",
+        survey: "Survey",
+        scenario: "Scenario",
         model: Type["LanguageModel"],
         debug: Optional[bool] = False,
         iteration: int = 0,
         cache: Optional["Cache"] = None,
         sidecar_model: Optional["LanguageModel"] = None,
-        skip_retry: bool = False,
-        raise_validation_errors: bool = True,
+        skip_retry=False,
     ):
         """Initialize the Interview instance.
 
@@ -110,15 +84,11 @@ class Interview(InterviewStatusMixin):
         ] = Answers()  # will get filled in as interview progresses
         self.sidecar_model = sidecar_model
 
-        # self.stop_on_exception = False
-
         # Trackers
         self.task_creators = TaskCreators()  # tracks the task creators
         self.exceptions = InterviewExceptionCollection()
-
         self._task_status_log_dict = InterviewStatusLog()
         self.skip_retry = skip_retry
-        self.raise_validation_errors = raise_validation_errors
 
         # dictionary mapping question names to their index in the survey.
         self.to_index = {
@@ -126,9 +96,6 @@ class Interview(InterviewStatusMixin):
             for index, question_name in enumerate(self.survey.question_names)
         }
 
-        self.failed_questions = []
-
-    # region: Serialization
     def _to_dict(self, include_exceptions=False) -> dict[str, Any]:
         """Return a dictionary representation of the Interview instance.
         This is just for hashing purposes.
@@ -153,301 +120,13 @@ class Interview(InterviewStatusMixin):
 
         return dict_hash(self._to_dict())
 
-    # endregion
-
-    # region: Creating tasks
-    @property
-    def dag(self) -> "DAG":
-        """Return the directed acyclic graph for the survey.
-
-        The DAG, or directed acyclic graph, is a dictionary that maps question names to their dependencies.
-        It is used to determine the order in which questions should be answered.
-        This reflects both agent 'memory' considerations and 'skip' logic.
-        The 'textify' parameter is set to True, so that the question names are returned as strings rather than integer indices.
-
-        >>> i = Interview.example()
-        >>> i.dag == {'q2': {'q0'}, 'q1': {'q0'}}
-        True
-        """
-        return self.survey.dag(textify=True)
-
-    def _build_question_tasks(
-        self,
-        model_buckets: ModelBuckets,
-    ) -> list[asyncio.Task]:
-        """Create a task for each question, with dependencies on the questions that must be answered before this one can be answered.
-
-        :param debug: whether to use debug mode, in which case `InvigilatorDebug` is used.
-        :param model_buckets: the model buckets used to track and control usage rates.
-        """
-        tasks = []
-        for question in self.survey.questions:
-            tasks_that_must_be_completed_before = list(
-                self._get_tasks_that_must_be_completed_before(
-                    tasks=tasks, question=question
-                )
-            )
-            question_task = self._create_question_task(
-                question=question,
-                tasks_that_must_be_completed_before=tasks_that_must_be_completed_before,
-                model_buckets=model_buckets,
-                iteration=self.iteration,
-            )
-            tasks.append(question_task)
-        return tuple(tasks)
-
-    def _get_tasks_that_must_be_completed_before(
-        self, *, tasks: list[asyncio.Task], question: "QuestionBase"
-    ) -> Generator[asyncio.Task, None, None]:
-        """Return the tasks that must be completed before the given question can be answered.
-
-        :param tasks: a list of tasks that have been created so far.
-        :param question: the question for which we are determining dependencies.
-
-        If a question has no dependencies, this will be an empty list, [].
-        """
-        parents_of_focal_question = self.dag.get(question.question_name, [])
-        for parent_question_name in parents_of_focal_question:
-            yield tasks[self.to_index[parent_question_name]]
-
-    def _create_question_task(
-        self,
-        *,
-        question: QuestionBase,
-        tasks_that_must_be_completed_before: list[asyncio.Task],
-        model_buckets: ModelBuckets,
-        iteration: int = 0,
-    ) -> asyncio.Task:
-        """Create a task that depends on the passed-in dependencies that are awaited before the task is run.
-
-        :param question: the question to be answered. This is the question we are creating a task for.
-        :param tasks_that_must_be_completed_before: the tasks that must be completed before the focal task is run.
-        :param model_buckets: the model buckets used to track and control usage rates.
-        :param debug: whether to use debug mode, in which case `InvigilatorDebug` is used.
-        :param iteration: the iteration number for the interview.
-
-        The task is created by a `QuestionTaskCreator`, which is responsible for creating the task and managing its dependencies.
-        It is passed a reference to the function that will be called to answer the question.
-        It is passed a list "tasks_that_must_be_completed_before" that are awaited before the task is run.
-        These are added as a dependency to the focal task.
-        """
-        task_creator = QuestionTaskCreator(
-            question=question,
-            answer_question_func=self._answer_question_and_record_task,
-            token_estimator=self._get_estimated_request_tokens,
-            model_buckets=model_buckets,
-            iteration=iteration,
-        )
-        for task in tasks_that_must_be_completed_before:
-            task_creator.add_dependency(task)
-
-        self.task_creators.update(
-            {question.question_name: task_creator}
-        )  # track this task creator
-        return task_creator.generate_task()
-
-    def _get_estimated_request_tokens(self, question) -> float:
-        """Estimate the number of tokens that will be required to run the focal task."""
-        invigilator = self._get_invigilator(question=question)
-        # TODO: There should be a way to get a more accurate estimate.
-        combined_text = ""
-        for prompt in invigilator.get_prompts().values():
-            if hasattr(prompt, "text"):
-                combined_text += prompt.text
-            elif isinstance(prompt, str):
-                combined_text += prompt
-            else:
-                raise ValueError(f"Prompt is of type {type(prompt)}")
-        return len(combined_text) / 4.0
-
-    async def _answer_question_and_record_task(
-        self,
-        *,
-        question: "QuestionBase",
-        task=None,
-    ) -> "AgentResponseDict":
-        """Answer a question and records the task."""
-
-        had_language_model_no_response_error = False
-
-        @retry(
-            stop=stop_after_attempt(EDSL_MAX_ATTEMPTS),
-            wait=wait_exponential(
-                multiplier=EDSL_BACKOFF_START_SEC, max=EDSL_BACKOFF_MAX_SEC
-            ),
-            retry=retry_if_exception_type(LanguageModelNoResponseError),
-            reraise=True,
-        )
-        async def attempt_answer():
-            nonlocal had_language_model_no_response_error
-
-            invigilator = self._get_invigilator(question)
-
-            if self._skip_this_question(question):
-                return invigilator.get_failed_task_result(
-                    failure_reason="Question skipped."
-                )
-
-            try:
-                response: EDSLResultObjectInput = (
-                    await invigilator.async_answer_question()
-                )
-                if response.validated:
-                    self.answers.add_answer(response=response, question=question)
-                    self._cancel_skipped_questions(question)
-                else:
-                    if (
-                        hasattr(response, "exception_occurred")
-                        and response.exception_occurred
-                    ):
-                        raise response.exception_occurred
-
-            except QuestionAnswerValidationError as e:
-                self._handle_exception(e, invigilator, task)
-                return invigilator.get_failed_task_result(
-                    failure_reason="Question answer validation failed."
-                )
-
-            except asyncio.TimeoutError as e:
-                self._handle_exception(e, invigilator, task)
-                had_language_model_no_response_error = True
-                raise LanguageModelNoResponseError(
-                    f"Language model timed out for question '{question.question_name}.'"
-                )
-
-            except Exception as e:
-                self._handle_exception(e, invigilator, task)
-
-            if "response" not in locals():
-                had_language_model_no_response_error = True
-                raise LanguageModelNoResponseError(
-                    f"Language model did not return a response for question '{question.question_name}.'"
-                )
-
-            # if it gets here, it means the no response error was fixed
-            if (
-                question.question_name in self.exceptions
-                and had_language_model_no_response_error
-            ):
-                self.exceptions.record_fixed_question(question.question_name)
-
-            return response
-
-        try:
-            return await attempt_answer()
-        except RetryError as retry_error:
-            # All retries have failed for LanguageModelNoResponseError
-            original_error = retry_error.last_attempt.exception()
-            self._handle_exception(
-                original_error, self._get_invigilator(question), task
-            )
-            raise original_error  # Re-raise the original error after handling
-
-    def _get_invigilator(self, question: QuestionBase) -> InvigilatorBase:
-        """Return an invigilator for the given question.
-
-        :param question: the question to be answered
-        :param debug: whether to use debug mode, in which case `InvigilatorDebug` is used.
-        """
-        invigilator = self.agent.create_invigilator(
-            question=question,
-            scenario=self.scenario,
-            model=self.model,
-            debug=False,
-            survey=self.survey,
-            memory_plan=self.survey.memory_plan,
-            current_answers=self.answers,
-            iteration=self.iteration,
-            cache=self.cache,
-            sidecar_model=self.sidecar_model,
-            raise_validation_errors=self.raise_validation_errors,
-        )
-        """Return an invigilator for the given question."""
-        return invigilator
-
-    def _skip_this_question(self, current_question: "QuestionBase") -> bool:
-        """Determine if the current question should be skipped.
-
-        :param current_question: the question to be answered.
-        """
-        current_question_index = self.to_index[current_question.question_name]
-
-        answers = self.answers | self.scenario | self.agent["traits"]
-        skip = self.survey.rule_collection.skip_question_before_running(
-            current_question_index, answers
-        )
-        return skip
-
-    def _handle_exception(
-        self, e: Exception, invigilator: "InvigilatorBase", task=None
-    ):
-        import copy
-
-        # breakpoint()
-
-        answers = copy.copy(self.answers)
-        exception_entry = InterviewExceptionEntry(
-            exception=e,
-            invigilator=invigilator,
-            answers=answers,
-        )
-        if task:
-            task.task_status = TaskStatus.FAILED
-        self.exceptions.add(invigilator.question.question_name, exception_entry)
-
-        if self.raise_validation_errors:
-            if isinstance(e, QuestionAnswerValidationError):
-                raise e
-
-        if hasattr(self, "stop_on_exception"):
-            stop_on_exception = self.stop_on_exception
-        else:
-            stop_on_exception = False
-
-        if stop_on_exception:
-            raise e
-
-    def _cancel_skipped_questions(self, current_question: QuestionBase) -> None:
-        """Cancel the tasks for questions that are skipped.
-
-        :param current_question: the question that was just answered.
-
-        It first determines the next question, given the current question and the current answers.
-        If the next question is the end of the survey, it cancels all remaining tasks.
-        If the next question is after the current question, it cancels all tasks between the current question and the next question.
-        """
-        current_question_index: int = self.to_index[current_question.question_name]
-
-        next_question: Union[
-            int, EndOfSurvey
-        ] = self.survey.rule_collection.next_question(
-            q_now=current_question_index,
-            answers=self.answers | self.scenario | self.agent["traits"],
-        )
-
-        next_question_index = next_question.next_q
-
-        def cancel_between(start, end):
-            """Cancel the tasks between the start and end indices."""
-            for i in range(start, end):
-                self.tasks[i].cancel()
-
-        if next_question_index == EndOfSurvey:
-            cancel_between(current_question_index + 1, len(self.survey.questions))
-            return
-
-        if next_question_index > (current_question_index + 1):
-            cancel_between(current_question_index + 1, next_question_index)
-
-    # endregion
-
-    # region: Conducting the interview
     async def async_conduct_interview(
         self,
-        model_buckets: Optional[ModelBuckets] = None,
+        *,
+        model_buckets: ModelBuckets = None,
+        debug: bool = False,
         stop_on_exception: bool = False,
         sidecar_model: Optional["LanguageModel"] = None,
-        raise_validation_errors: bool = True,
     ) -> tuple["Answers", List[dict[str, Any]]]:
         """
         Conduct an Interview asynchronously.
@@ -467,6 +146,19 @@ class Interview(InterviewStatusMixin):
 
         >>> i = Interview.example(throw_exception = True)
         >>> result, _ = asyncio.run(i.async_conduct_interview())
+        Attempt 1 failed with exception:This is a test error now waiting 1.00 seconds before retrying.Parameters: start=1.0, max=60.0, max_attempts=5.
+        <BLANKLINE>
+        <BLANKLINE>
+        Attempt 2 failed with exception:This is a test error now waiting 2.00 seconds before retrying.Parameters: start=1.0, max=60.0, max_attempts=5.
+        <BLANKLINE>
+        <BLANKLINE>
+        Attempt 3 failed with exception:This is a test error now waiting 4.00 seconds before retrying.Parameters: start=1.0, max=60.0, max_attempts=5.
+        <BLANKLINE>
+        <BLANKLINE>
+        Attempt 4 failed with exception:This is a test error now waiting 8.00 seconds before retrying.Parameters: start=1.0, max=60.0, max_attempts=5.
+        <BLANKLINE>
+        <BLANKLINE>
+
         >>> i.exceptions
         {'q0': ...
         >>> i = Interview.example()
@@ -476,30 +168,26 @@ class Interview(InterviewStatusMixin):
         asyncio.exceptions.CancelledError
         """
         self.sidecar_model = sidecar_model
-        self.stop_on_exception = stop_on_exception
 
         # if no model bucket is passed, create an 'infinity' bucket with no rate limits
         if model_buckets is None or hasattr(self.agent, "answer_question_directly"):
             model_buckets = ModelBuckets.infinity_bucket()
 
+        ## build the tasks using the InterviewTaskBuildingMixin
         ## This is the key part---it creates a task for each question,
         ## with dependencies on the questions that must be answered before this one can be answered.
-        self.tasks = self._build_question_tasks(model_buckets=model_buckets)
+        self.tasks = self._build_question_tasks(
+            debug=debug, model_buckets=model_buckets
+        )
 
         ## 'Invigilators' are used to administer the survey
-        self.invigilators = [
-            self._get_invigilator(question) for question in self.survey.questions
-        ]
-        await asyncio.gather(
-            *self.tasks, return_exceptions=not stop_on_exception
-        )  # not stop_on_exception)
+        self.invigilators = list(self._build_invigilators(debug=debug))
+        # await the tasks being conducted
+        await asyncio.gather(*self.tasks, return_exceptions=not stop_on_exception)
         self.answers.replace_missing_answers_with_none(self.survey)
         valid_results = list(self._extract_valid_results())
         return self.answers, valid_results
 
-    # endregion
-
-    # region: Extracting results and recording errors
     def _extract_valid_results(self) -> Generator["Answers", None, None]:
         """Extract the valid results from the list of results.
 
@@ -512,6 +200,8 @@ class Interview(InterviewStatusMixin):
         >>> results = list(i._extract_valid_results())
         >>> len(results) == len(i.survey)
         True
+        >>> type(results[0])
+        <class 'edsl.data_transfer_models.AgentResponseDict'>
         """
         assert len(self.tasks) == len(self.invigilators)
 
@@ -522,24 +212,46 @@ class Interview(InterviewStatusMixin):
             try:
                 result = task.result()
             except asyncio.CancelledError as e:  # task was cancelled
-                result = invigilator.get_failed_task_result(
-                    failure_reason="Task was cancelled."
-                )
+                result = invigilator.get_failed_task_result()
             except Exception as e:  # any other kind of exception in the task
-                result = invigilator.get_failed_task_result(
-                    failure_reason=f"Task failed with exception: {str(e)}."
-                )
-                exception_entry = InterviewExceptionEntry(
-                    exception=e,
-                    invigilator=invigilator,
-                )
-                self.exceptions.add(task.get_name(), exception_entry)
-
+                result = invigilator.get_failed_task_result()
+                self._record_exception(task, e)
             yield result
 
-    # endregion
+    def _record_exception(self, task, exception: Exception) -> None:
+        """Record an exception in the Interview instance.
+
+        It records the exception in the Interview instance, with the task name and the exception entry.
+
+        >>> i = Interview.example()
+        >>> result, _ = asyncio.run(i.async_conduct_interview())
+        >>> i.exceptions
+        {}
+        >>> i._record_exception(i.tasks[0], Exception("An exception occurred."))
+        >>> i.exceptions
+        {'q0': ...
+        """
+        exception_entry = InterviewExceptionEntry(exception)
+        self.exceptions.add(task.get_name(), exception_entry)
+
+    @property
+    def dag(self) -> "DAG":
+        """Return the directed acyclic graph for the survey.
+
+        The DAG, or directed acyclic graph, is a dictionary that maps question names to their dependencies.
+        It is used to determine the order in which questions should be answered.
+        This reflects both agent 'memory' considerations and 'skip' logic.
+        The 'textify' parameter is set to True, so that the question names are returned as strings rather than integer indices.
+
+        >>> i = Interview.example()
+        >>> i.dag == {'q2': {'q0'}, 'q1': {'q0'}}
+        True
+        """
+        return self.survey.dag(textify=True)
 
-    # region: Magic methods
+    #######################
+    # Dunder methods
+    #######################
     def __repr__(self) -> str:
         """Return a string representation of the Interview instance."""
         return f"Interview(agent = {repr(self.agent)}, survey = {repr(self.survey)}, scenario = {repr(self.scenario)}, model = {repr(self.model)})"

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -2,62 +2,24 @@ import traceback
 import datetime
 import time
 from collections import UserDict
-from edsl.jobs.FailedQuestion import FailedQuestion
+
+# traceback=traceback.format_exc(),
+# traceback = frame_summary_to_dict(traceback.extract_tb(e.__traceback__))
+# traceback = [frame_summary_to_dict(f) for f in traceback.extract_tb(e.__traceback__)]
 
 
 class InterviewExceptionEntry:
-    """Class to record an exception that occurred during the interview."""
-
-    def __init__(
-        self,
-        *,
-        exception: Exception,
-        # failed_question: FailedQuestion,
-        invigilator: "Invigilator",
-        traceback_format="text",
-        answers=None,
-    ):
+    """Class to record an exception that occurred during the interview.
+
+    >>> entry = InterviewExceptionEntry.example()
+    >>> entry.to_dict()['exception']
+    "ValueError('An error occurred.')"
+    """
+
+    def __init__(self, exception: Exception, traceback_format="html"):
         self.time = datetime.datetime.now().isoformat()
         self.exception = exception
-        # self.failed_question = failed_question
-        self.invigilator = invigilator
         self.traceback_format = traceback_format
-        self.answers = answers
-
-    @property
-    def question_type(self):
-        # return self.failed_question.question.question_type
-        return self.invigilator.question.question_type
-
-    @property
-    def name(self):
-        return repr(self.exception)
-
-    @property
-    def rendered_prompts(self):
-        return self.invigilator.get_prompts()
-
-    @property
-    def key_sequence(self):
-        return self.invigilator.model.key_sequence
-
-    @property
-    def generated_token_string(self):
-        # return "POO"
-        if self.invigilator.raw_model_response is None:
-            return "No raw model response available."
-        else:
-            return self.invigilator.model.get_generated_token_string(
-                self.invigilator.raw_model_response
-            )
-
-    @property
-    def raw_model_response(self):
-        import json
-
-        if self.invigilator.raw_model_response is None:
-            return "No raw model response available."
-        return json.dumps(self.invigilator.raw_model_response, indent=2)
 
     def __getitem__(self, key):
         # Support dict-like access obj['a']
@@ -65,37 +27,11 @@ class InterviewExceptionEntry:
 
     @classmethod
     def example(cls):
-        from edsl import QuestionFreeText
-        from edsl.language_models import LanguageModel
-
-        m = LanguageModel.example(test_model=True)
-        q = QuestionFreeText.example(exception_to_throw=ValueError)
-        results = q.by(m).run(
-            skip_retry=True, print_exceptions=False, raise_validation_errors=True
-        )
-        return results.task_history.exceptions[0]["how_are_you"][0]
-
-    @property
-    def code_to_reproduce(self):
-        return self.code(run=False)
-
-    def code(self, run=True):
-        lines = []
-        lines.append("from edsl import Question, Model, Scenario, Agent")
-
-        lines.append(f"q = {repr(self.invigilator.question)}")
-        lines.append(f"scenario = {repr(self.invigilator.scenario)}")
-        lines.append(f"agent = {repr(self.invigilator.agent)}")
-        lines.append(f"m = Model('{self.invigilator.model.model}')")
-        lines.append("results = q.by(m).by(agent).by(scenario).run()")
-        code_str = "\n".join(lines)
-
-        if run:
-            # Create a new namespace to avoid polluting the global namespace
-            namespace = {}
-            exec(code_str, namespace)
-            return namespace["results"]
-        return code_str
+        try:
+            raise ValueError("An error occurred.")
+        except Exception as e:
+            entry = InterviewExceptionEntry(e)
+        return entry
 
     @property
     def traceback(self):
@@ -142,15 +78,13 @@ class InterviewExceptionEntry:
 
         >>> entry = InterviewExceptionEntry.example()
         >>> entry.to_dict()['exception']
-        ValueError()
+        "ValueError('An error occurred.')"
 
         """
         return {
-            "exception": self.exception,
+            "exception": repr(self.exception),
             "time": self.time,
             "traceback": self.traceback,
-            # "failed_question": self.failed_question.to_dict(),
-            "invigilator": self.invigilator.to_dict(),
         }
 
     def push(self):