edsl 0.1.46__py3-none-any.whl → 0.1.48__py3-none-any.whl

edsl/{jobs/AnswerQuestionFunctionConstructor.py → interviews/answering_function.py}

@@ -1,26 +1,26 @@
 import copy
 import asyncio
 
-from typing import Union, Type, Callable, TYPE_CHECKING
+from typing import Union, Type, Callable, TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
-    from edsl.questions.QuestionBase import QuestionBase
-    from edsl.jobs.interviews.Interview import Interview
-    from edsl.language_models.key_management.KeyLookup import KeyLookup
+    from ..questions import QuestionBase
+    from .interview import Interview
+    from ..key_management import KeyLookup
 
-from edsl.surveys.base import EndOfSurvey
-from edsl.jobs.tasks.task_status_enum import TaskStatus
+from ..surveys.base import EndOfSurvey
+from ..tasks import TaskStatus
 
-from edsl.jobs.FetchInvigilator import FetchInvigilator
-from edsl.exceptions.language_models import LanguageModelNoResponseError
-from edsl.exceptions.questions import QuestionAnswerValidationError
-from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
+from ..jobs.fetch_invigilator import FetchInvigilator
+from ..language_models.exceptions import LanguageModelNoResponseError
+from ..questions.exceptions import QuestionAnswerValidationError
+from ..data_transfer_models import AgentResponseDict, EDSLResultObjectInput, Answers
 
-from edsl.jobs.Answers import Answers
+from .exception_tracking import InterviewExceptionEntry
 
 
 class RetryConfig:
-    from edsl.config import CONFIG
+    from ..config import CONFIG
 
     EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
     EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
@@ -46,17 +46,45 @@ class SkipHandler:
             | self.interview.agent["traits"]
         )
         return self.skip_function(current_question_index, combined_answers)
+    
+    def _current_info_env(self) -> dict[str, Any]:
+        """
+        - The current answers are "generated_tokens" and "comment" 
+        - The scenario should have "scenario." added to the keys
+        - The agent traits should have "agent." added to the keys
+        """
+        # Process answers dictionary
+        processed_answers = {}
+        for key, value in self.interview.answers.items():
+            if key.endswith("_generated_tokens"):
+                base_name = key.replace("_generated_tokens", "")
+                processed_answers[f"{base_name}.generated_tokens"] = value
+            elif key.endswith("_comment"):
+                base_name = key.replace("_comment", "")
+                processed_answers[f"{base_name}.comment"] = value
+            else:
+                # Regular answer
+                processed_answers[f"{key}.answer"] = value
+
+        # Process scenario dictionary
+        processed_scenario = {f"scenario.{k}": v for k, v in self.interview.scenario.items()}
+
+        # Process agent traits
+        processed_agent = {f"agent.{k}": v for k, v in self.interview.agent["traits"].items()}
+
+        return processed_answers | processed_scenario | processed_agent
 
     def cancel_skipped_questions(self, current_question: "QuestionBase") -> None:
         """Cancel the tasks for questions that should be skipped."""
         current_question_index: int = self.interview.to_index[
             current_question.question_name
         ]
-        answers = (
-            self.interview.answers
-            | self.interview.scenario
-            | self.interview.agent["traits"]
-        )
+        answers = self._current_info_env()
+        # answers = (
+        #     self.interview.answers
+        #     | self.interview.scenario
+        #     | self.interview.agent["traits"]
+        # )
 
         # Get the index of the next question, which could also be the end of the survey
         next_question: Union[int, EndOfSurvey] = (
@@ -111,7 +139,6 @@ class AnswerQuestionFunctionConstructor:
     ):
         """Handle an exception that occurred while answering a question."""
 
-        from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 
         answers = copy.copy(
             self.interview.answers

edsl/{jobs/interviews/InterviewExceptionEntry.py → interviews/exception_tracking.py}

@@ -1,7 +1,9 @@
+from collections import UserDict
 import traceback
 import datetime
-from edsl.agents.InvigilatorBase import InvigilatorBase
+import json
 
+from ..invigilators import InvigilatorBase
 
 class InterviewExceptionEntry:
     """Class to record an exception that occurred during the interview."""
@@ -16,33 +18,33 @@ class InterviewExceptionEntry:
     ):
         self.time = datetime.datetime.now().isoformat()
         self.exception = exception
-        # self.failed_question = failed_question
         self.invigilator = invigilator
         self.traceback_format = traceback_format
         self.answers = answers
 
-        # breakpoint()
-
     @property
-    def question_type(self):
-        # return self.failed_question.question.question_type
+    def question_type(self) -> str:
+        """Return the type of the question that failed."""
         return self.invigilator.question.question_type
 
     @property
-    def name(self):
+    def name(self) -> str:
+        """Return the name of the exception."""
         return repr(self.exception)
 
     @property
-    def rendered_prompts(self):
+    def rendered_prompts(self) -> str:
+        """Return the rendered prompts."""
         return self.invigilator.get_prompts()
 
     @property
-    def key_sequence(self):
+    def key_sequence(self) -> tuple[str, ...]:
+        """Return the key sequence."""
         return self.invigilator.model.key_sequence
 
     @property
-    def generated_token_string(self):
-        # return "POO"
+    def generated_token_string(self) -> str:
+        """Return the generated token string."""
         if self.invigilator.raw_model_response is None:
             return "No raw model response available."
         else:
@@ -51,9 +53,8 @@ class InterviewExceptionEntry:
             )
 
     @property
-    def raw_model_response(self):
-        import json
-
+    def raw_model_response(self) -> dict:
+        """Return the raw model response."""
         if self.invigilator.raw_model_response is None:
             return "No raw model response available."
         return json.dumps(self.invigilator.raw_model_response, indent=2)
@@ -64,8 +65,12 @@ class InterviewExceptionEntry:
 
     @classmethod
     def example(cls):
-        from edsl import QuestionFreeText
-        from edsl.language_models import LanguageModel
+        """Return an example InterviewExceptionEntry.
+
+        >>> entry = InterviewExceptionEntry.example()
+        """
+        from ..questions import QuestionFreeText
+        from ..language_models import LanguageModel
 
         m = LanguageModel.example(test_model=True)
         q = QuestionFreeText.example(exception_to_throw=ValueError)
@@ -80,9 +85,11 @@ class InterviewExceptionEntry:
 
     @property
     def code_to_reproduce(self):
+        """Return the code to reproduce the exception."""
         return self.code(run=False)
 
     def code(self, run=True):
+        """Return the code to reproduce the exception."""
         lines = []
         lines.append("from edsl import Question, Model, Scenario, Agent")
 
@@ -101,7 +108,7 @@ class InterviewExceptionEntry:
         return code_str
 
     @property
-    def traceback(self):
+    def traceback(self) -> str:
         """Return the exception as HTML."""
         if self.traceback_format == "html":
             return self.html_traceback
@@ -109,7 +116,7 @@ class InterviewExceptionEntry:
             return self.text_traceback
 
     @property
-    def text_traceback(self):
+    def text_traceback(self) -> str:
         """
         >>> entry = InterviewExceptionEntry.example()
         >>> entry.text_traceback
@@ -120,7 +127,7 @@ class InterviewExceptionEntry:
         return tb_str
 
     @property
-    def html_traceback(self):
+    def html_traceback(self) -> str:
         from rich.console import Console
         from rich.table import Table
         from rich.traceback import Traceback
@@ -142,6 +149,11 @@ class InterviewExceptionEntry:
 
     @staticmethod
     def serialize_exception(exception: Exception) -> dict:
+        """Serialize an exception to a dictionary.
+
+        >>> entry = InterviewExceptionEntry.example()
+        >>> _ = entry.serialize_exception(entry.exception)
+        """
         return {
             "type": type(exception).__name__,
             "message": str(exception),
@@ -154,6 +166,11 @@ class InterviewExceptionEntry:
 
     @staticmethod
     def deserialize_exception(data: dict) -> Exception:
+        """Deserialize an exception from a dictionary.
+
+        >>> entry = InterviewExceptionEntry.example()
+        >>> _ = entry.deserialize_exception(entry.to_dict()["exception"])
+        """
         try:
             exception_class = globals()[data["type"]]
         except KeyError:
@@ -166,8 +183,7 @@ class InterviewExceptionEntry:
         >>> entry = InterviewExceptionEntry.example()
         >>> _ = entry.to_dict()
         """
-        import json
-        from edsl.exceptions.questions import QuestionAnswerValidationError
+        from ..questions.exceptions import QuestionAnswerValidationError
 
         invigilator = (
             self.invigilator.to_dict() if self.invigilator is not None else None
@@ -192,7 +208,7 @@ class InterviewExceptionEntry:
     @classmethod
     def from_dict(cls, data: dict) -> "InterviewExceptionEntry":
         """Create an InterviewExceptionEntry from a dictionary."""
-        from edsl.agents.Invigilator import InvigilatorAI
+        from ..invigilators import InvigilatorAI
 
         exception = cls.deserialize_exception(data["exception"])
         if data["invigilator"] is None:
@@ -202,6 +218,75 @@ class InterviewExceptionEntry:
         return cls(exception=exception, invigilator=invigilator)
 
 
+
+class InterviewExceptionCollection(UserDict):
+    """A collection of exceptions that occurred during the interview."""
+
+    def __init__(self):
+        """Initialize the InterviewExceptionCollection."""
+        super().__init__()
+        self.fixed = set()
+
+    def unfixed_exceptions(self) -> list:
+        """Return a list of unfixed exceptions."""
+        return {k: v for k, v in self.data.items() if k not in self.fixed}
+
+    def num_unfixed(self) -> int:
+        """Return a list of unfixed questions."""
+        return len([k for k in self.data.keys() if k not in self.fixed])
+
+    def record_fixed_question(self, question_name: str) -> None:
+        """Record that a question has been fixed."""
+        self.fixed.add(question_name)
+
+    def add(self, question_name: str, entry: InterviewExceptionEntry) -> None:
+        """Add an exception entry to the collection."""
+        question_name = question_name
+        if question_name not in self.data:
+            self.data[question_name] = []
+        self.data[question_name].append(entry)
+
+    def to_dict(self, include_traceback=True) -> dict:
+        """Return the collection of exceptions as a dictionary."""
+        newdata = {k: [e.to_dict() for e in v] for k, v in self.data.items()}
+        return newdata
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "InterviewExceptionCollection":
+        """Create an InterviewExceptionCollection from a dictionary."""
+        collection = cls()
+        for question_name, entries in data.items():
+            for entry in entries:
+                collection.add(question_name, InterviewExceptionEntry.from_dict(entry))
+        return collection
+
+    def _repr_html_(self) -> str:
+        from ..utilities.utilities import data_to_html
+
+        return data_to_html(self.to_dict(include_traceback=True))
+
+    def ascii_table(self, traceback: bool = False) -> None:
+        """Print the collection of exceptions as an ASCII table."""
+        headers = ["Question name", "Exception", "Time", "Traceback"]
+        from tabulate import tabulate
+
+        data = []
+        for question, exceptions in self.data.items():
+            for exception in exceptions:
+                if traceback:
+                    row = [
+                        question,
+                        exception["exception"],
+                        exception["time"],
+                        exception["traceback"],
+                    ]
+                else:
+                    row = [question, exception["exception"], exception["time"]]
+                data.append(row)
+
+        print(tabulate(data, headers=headers, tablefmt="grid"))
+
+
 if __name__ == "__main__":
     import doctest