edsl 0.1.39.dev2__py3-none-any.whl → 0.1.39.dev4__py3-none-any.whl

edsl/jobs/data_structures.py

@@ -0,0 +1,120 @@
+from typing import Optional, Literal
+from dataclasses import dataclass, asdict
+
+# from edsl.data_transfer_models import VisibilityType
+from edsl.data.Cache import Cache
+from edsl.jobs.buckets.BucketCollection import BucketCollection
+from edsl.language_models.key_management.KeyLookup import KeyLookup
+from edsl.jobs.runners.JobsRunnerStatus import JobsRunnerStatus
+
+VisibilityType = Literal["private", "public", "unlisted"]
+from edsl.Base import Base
+
+
+@dataclass
+class RunEnvironment:
+    cache: Optional[Cache] = None
+    bucket_collection: Optional[BucketCollection] = None
+    key_lookup: Optional[KeyLookup] = None
+    jobs_runner_status: Optional["JobsRunnerStatus"] = None
+
+
+@dataclass
+class RunParameters(Base):
+    n: int = 1
+    progress_bar: bool = False
+    stop_on_exception: bool = False
+    check_api_keys: bool = False
+    verbose: bool = True
+    print_exceptions: bool = True
+    remote_cache_description: Optional[str] = None
+    remote_inference_description: Optional[str] = None
+    remote_inference_results_visibility: Optional[VisibilityType] = "unlisted"
+    skip_retry: bool = False
+    raise_validation_errors: bool = False
+    disable_remote_cache: bool = False
+    disable_remote_inference: bool = False
+    job_uuid: Optional[str] = None
+
+    def to_dict(self, add_edsl_version=False) -> dict:
+        d = asdict(self)
+        if add_edsl_version:
+            from edsl import __version__
+
+            d["edsl_version"] = __version__
+            d["edsl_class_name"] = "RunConfig"
+        return d
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "RunConfig":
+        return cls(**data)
+
+    def code(self):
+        return f"RunConfig(**{self.to_dict()})"
+
+    @classmethod
+    def example(cls) -> "RunConfig":
+        return cls()
+
+
+@dataclass
+class RunConfig:
+    environment: RunEnvironment
+    parameters: RunParameters
+
+    def add_environment(self, environment: RunEnvironment):
+        self.environment = environment
+
+    def add_bucket_collection(self, bucket_collection: BucketCollection):
+        self.environment.bucket_collection = bucket_collection
+
+    def add_cache(self, cache: Cache):
+        self.environment.cache = cache
+
+    def add_key_lookup(self, key_lookup: KeyLookup):
+        self.environment.key_lookup = key_lookup
+
+
+"""This module contains the Answers class, which is a helper class to hold the answers to a survey."""
+
+from collections import UserDict
+from edsl.data_transfer_models import EDSLResultObjectInput
+
+
+class Answers(UserDict):
+    """Helper class to hold the answers to a survey."""
+
+    def add_answer(
+        self, response: EDSLResultObjectInput, question: "QuestionBase"
+    ) -> None:
+        """Add a response to the answers dictionary."""
+        answer = response.answer
+        comment = response.comment
+        generated_tokens = response.generated_tokens
+        # record the answer
+        if generated_tokens:
+            self[question.question_name + "_generated_tokens"] = generated_tokens
+        self[question.question_name] = answer
+        if comment:
+            self[question.question_name + "_comment"] = comment
+
+    def replace_missing_answers_with_none(self, survey: "Survey") -> None:
+        """Replace missing answers with None. Answers can be missing if the agent skips a question."""
+        for question_name in survey.question_names:
+            if question_name not in self:
+                self[question_name] = None
+
+    def to_dict(self):
+        """Return a dictionary of the answers."""
+        return self.data
+
+    @classmethod
+    def from_dict(cls, d):
+        """Return an Answers object from a dictionary."""
+        return cls(d)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

edsl/jobs/interviews/Interview.py

@@ -4,10 +4,10 @@ from __future__ import annotations
 import asyncio
 from typing import Any, Type, List, Generator, Optional, Union, TYPE_CHECKING
 import copy
+from dataclasses import dataclass
 
-# from edsl.config import CONFIG
-
-from edsl.jobs.Answers import Answers
+# from edsl.jobs.Answers import Answers
+from edsl.jobs.data_structures import Answers
 from edsl.jobs.interviews.InterviewStatusLog import InterviewStatusLog
 from edsl.jobs.interviews.InterviewStatusDictionary import InterviewStatusDictionary
 from edsl.jobs.interviews.InterviewExceptionCollection import (
@@ -22,6 +22,7 @@ from edsl.jobs.InterviewTaskManager import InterviewTaskManager
 from edsl.jobs.FetchInvigilator import FetchInvigilator
 from edsl.jobs.RequestTokenEstimator import RequestTokenEstimator
 
+
 if TYPE_CHECKING:
     from edsl.agents.Agent import Agent
     from edsl.surveys.Survey import Survey
@@ -29,6 +30,16 @@ if TYPE_CHECKING:
     from edsl.data.Cache import Cache
     from edsl.language_models.LanguageModel import LanguageModel
     from edsl.jobs.tokens.InterviewTokenUsage import InterviewTokenUsage
+    from edsl.agents.InvigilatorBase import InvigilatorBase
+    from edsl.language_models.key_management.KeyLookup import KeyLookup
+
+
+@dataclass
+class InterviewRunningConfig:
+    cache: Optional["Cache"] = (None,)
+    skip_retry: bool = (False,)  # COULD BE SET WITH CONFIG
+    raise_validation_errors: bool = (True,)
+    stop_on_exception: bool = (False,)
 
 
 class Interview:
@@ -45,13 +56,11 @@ class Interview:
         survey: Survey,
         scenario: Scenario,
         model: Type["LanguageModel"],
-        debug: Optional[bool] = False,  # DEPRECATE
         iteration: int = 0,
+        indices: dict = None,  # explain?
         cache: Optional["Cache"] = None,
-        sidecar_model: Optional["LanguageModel"] = None,  # DEPRECATE
         skip_retry: bool = False,  # COULD BE SET WITH CONFIG
         raise_validation_errors: bool = True,
-        indices: dict = None,  # explain?
     ):
         """Initialize the Interview instance.
 
@@ -59,10 +68,9 @@ class Interview:
         :param survey: the survey being administered to the agent.
         :param scenario: the scenario that populates the survey questions.
         :param model: the language model used to answer the questions.
-        :param debug: if True, run without calls to the language model.
+        # :param debug: if True, run without calls to the language model.
         :param iteration: the iteration number of the interview.
         :param cache: the cache used to store the answers.
-        :param sidecar_model: a sidecar model used to answer questions.
 
         >>> i = Interview.example()
         >>> i.task_manager.task_creators
@@ -83,12 +91,9 @@ class Interview:
         self.survey = copy.deepcopy(survey)  # why do we need to deepcopy the survey?
         self.scenario = scenario
         self.model = model
-        self.debug = debug
         self.iteration = iteration
-        self.cache = cache
 
         self.answers = Answers()  # will get filled in as interview progresses
-        self.sidecar_model = sidecar_model
 
         self.task_manager = InterviewTaskManager(
             survey=self.survey,
@@ -97,6 +102,13 @@ class Interview:
 
         self.exceptions = InterviewExceptionCollection()
 
+        self.running_config = InterviewRunningConfig(
+            cache=cache,
+            skip_retry=skip_retry,
+            raise_validation_errors=raise_validation_errors,
+        )
+
+        self.cache = cache
         self.skip_retry = skip_retry
         self.raise_validation_errors = raise_validation_errors
 
@@ -109,6 +121,7 @@ class Interview:
         self.failed_questions = []
 
         self.indices = indices
+        self.initial_hash = hash(self)
 
     @property
     def has_exceptions(self) -> bool:
@@ -134,7 +147,6 @@ class Interview:
         # return self.task_creators.interview_status
         return self.task_manager.interview_status
 
-    # region: Serialization
     def to_dict(self, include_exceptions=True, add_edsl_version=True) -> dict[str, Any]:
         """Return a dictionary representation of the Interview instance.
         This is just for hashing purposes.
@@ -198,13 +210,13 @@ class Interview:
         """
         return hash(self) == hash(other)
 
-    # region: Conducting the interview
     async def async_conduct_interview(
         self,
-        model_buckets: Optional[ModelBuckets] = None,
-        stop_on_exception: bool = False,
-        sidecar_model: Optional["LanguageModel"] = None,
-        raise_validation_errors: bool = True,
+        run_config: Optional["RunConfig"] = None,
+        #     model_buckets: Optional[ModelBuckets] = None,
+        #     stop_on_exception: bool = False,
+        #     raise_validation_errors: bool = True,
+        #     key_lookup: Optional[KeyLookup] = None,
     ) -> tuple["Answers", List[dict[str, Any]]]:
         """
         Conduct an Interview asynchronously.
@@ -213,7 +225,6 @@ class Interview:
         :param model_buckets: a dictionary of token buckets for the model.
         :param debug: run without calls to LLM.
         :param stop_on_exception: if True, stops the interview if an exception is raised.
-        :param sidecar_model: a sidecar model used to answer questions.
 
         Example usage:
 
@@ -227,21 +238,39 @@ class Interview:
         >>> i.exceptions
         {'q0': ...
         >>> i = Interview.example()
-        >>> result, _ = asyncio.run(i.async_conduct_interview(stop_on_exception = True))
+        >>> from edsl.jobs.Jobs import RunConfig, RunParameters, RunEnvironment
+        >>> run_config = RunConfig(parameters = RunParameters(), environment = RunEnvironment())
+        >>> run_config.parameters.stop_on_exception = True
+        >>> result, _ = asyncio.run(i.async_conduct_interview(run_config))
         Traceback (most recent call last):
         ...
         asyncio.exceptions.CancelledError
         """
-        self.sidecar_model = sidecar_model
-        self.stop_on_exception = stop_on_exception
+        from edsl.jobs.Jobs import RunConfig, RunParameters, RunEnvironment
+
+        if run_config is None:
+            run_config = RunConfig(
+                parameters=RunParameters(),
+                environment=RunEnvironment(),
+            )
+        self.stop_on_exception = run_config.parameters.stop_on_exception
 
         # if no model bucket is passed, create an 'infinity' bucket with no rate limits
+        bucket_collection = run_config.environment.bucket_collection
+
+        if bucket_collection:
+            model_buckets = bucket_collection.get(self.model)
+        else:
+            model_buckets = None
+
         if model_buckets is None or hasattr(self.agent, "answer_question_directly"):
             model_buckets = ModelBuckets.infinity_bucket()
 
         # was "self.tasks" - is that necessary?
         self.tasks = self.task_manager.build_question_tasks(
-            answer_func=AnswerQuestionFunctionConstructor(self)(),
+            answer_func=AnswerQuestionFunctionConstructor(
+                self, key_lookup=run_config.environment.key_lookup
+            )(),
             token_estimator=RequestTokenEstimator(self),
             model_buckets=model_buckets,
         )
@@ -250,23 +279,26 @@ class Interview:
         ## with dependencies on the questions that must be answered before this one can be answered.
 
         ## 'Invigilators' are used to administer the survey.
-        self.invigilators = [
-            FetchInvigilator(interview=self, current_answers=self.answers)(question)
-            for question in self.survey.questions
-        ]
-        await asyncio.gather(*self.tasks, return_exceptions=not stop_on_exception)
+        fetcher = FetchInvigilator(
+            interview=self,
+            current_answers=self.answers,
+            key_lookup=run_config.environment.key_lookup,
+        )
+        self.invigilators = [fetcher(question) for question in self.survey.questions]
+        await asyncio.gather(
+            *self.tasks, return_exceptions=not run_config.parameters.stop_on_exception
+        )
         self.answers.replace_missing_answers_with_none(self.survey)
         valid_results = list(
             self._extract_valid_results(self.tasks, self.invigilators, self.exceptions)
         )
         return self.answers, valid_results
 
-    # endregion
-
-    # region: Extracting results and recording errors
     @staticmethod
     def _extract_valid_results(
-        tasks, invigilators: List["InvigilatorABC"], exceptions
+        tasks: List["asyncio.Task"],
+        invigilators: List["InvigilatorBase"],
+        exceptions: InterviewExceptionCollection,
     ) -> Generator["Answers", None, None]:
         """Extract the valid results from the list of results.
 
@@ -279,10 +311,7 @@ class Interview:
         """
         assert len(tasks) == len(invigilators)
 
-        for task, invigilator in zip(tasks, invigilators):
-            if not task.done():
-                raise ValueError(f"Task {task.get_name()} is not done.")
-
+        def handle_task(task, invigilator):
             try:
                 result = task.result()
             except asyncio.CancelledError as e:  # task was cancelled
@@ -298,17 +327,21 @@ class Interview:
                     invigilator=invigilator,
                 )
                 exceptions.add(task.get_name(), exception_entry)
+            return result
 
-            yield result
+        for task, invigilator in zip(tasks, invigilators):
+            if not task.done():
+                raise ValueError(f"Task {task.get_name()} is not done.")
 
-    # endregion
+            yield handle_task(task, invigilator)
 
-    # region: Magic methods
     def __repr__(self) -> str:
         """Return a string representation of the Interview instance."""
         return f"Interview(agent = {repr(self.agent)}, survey = {repr(self.survey)}, scenario = {repr(self.scenario)}, model = {repr(self.model)})"
 
-    def duplicate(self, iteration: int, cache: "Cache") -> Interview:
+    def duplicate(
+        self, iteration: int, cache: "Cache", randomize_survey: Optional[bool] = True
+    ) -> Interview:
         """Duplicate the interview, but with a new iteration number and cache.
 
         >>> i = Interview.example()
@@ -317,14 +350,19 @@ class Interview:
         True
 
         """
+        if randomize_survey:
+            new_survey = self.survey.draw()
+        else:
+            new_survey = self.survey
+
         return Interview(
             agent=self.agent,
-            survey=self.survey,
+            survey=new_survey,
             scenario=self.scenario,
             model=self.model,
             iteration=iteration,
-            cache=cache,
-            skip_retry=self.skip_retry,
+            cache=self.running_config.cache,
+            skip_retry=self.running_config.skip_retry,
             indices=self.indices,
         )
 

edsl/jobs/results_exceptions_handler.py

@@ -0,0 +1,98 @@
+from typing import Optional, TYPE_CHECKING, Protocol
+import sys
+from edsl.scenarios.FileStore import HTMLFileStore
+from edsl.config import CONFIG
+from edsl.coop.coop import Coop
+
+
+class ResultsProtocol(Protocol):
+    """Protocol defining the required interface for Results objects."""
+
+    @property
+    def has_unfixed_exceptions(self) -> bool: ...
+
+    @property
+    def task_history(self) -> "TaskHistoryProtocol": ...
+
+
+class TaskHistoryProtocol(Protocol):
+    """Protocol defining the required interface for TaskHistory objects."""
+
+    @property
+    def indices(self) -> list: ...
+
+    def html(self, cta: str, open_in_browser: bool, return_link: bool) -> str: ...
+
+
+class RunParametersProtocol(Protocol):
+    """Protocol defining the required interface for RunParameters objects."""
+
+    @property
+    def print_exceptions(self) -> bool: ...
+
+
+class ResultsExceptionsHandler:
+    """Handles exception reporting and display functionality."""
+
+    def __init__(
+        self, results: ResultsProtocol, parameters: RunParametersProtocol
+    ) -> None:
+        self.results = results
+        self.parameters = parameters
+
+        self.open_in_browser = self._get_browser_setting()
+        self.remote_logging = self._get_remote_logging_setting()
+
+    def _get_browser_setting(self) -> bool:
+        """Determine if exceptions should be opened in browser based on config."""
+        setting = CONFIG.get("EDSL_OPEN_EXCEPTION_REPORT_URL")
+        if setting == "True":
+            return True
+        elif setting == "False":
+            return False
+        else:
+            raise Exception(
+                "EDSL_OPEN_EXCEPTION_REPORT_URL must be either True or False"
+            )
+
+    def _get_remote_logging_setting(self) -> bool:
+        """Get remote logging setting from coop."""
+        try:
+            coop = Coop()
+            return coop.edsl_settings["remote_logging"]
+        except Exception as e:
+            # print(e)
+            return False
+
+    def _generate_error_message(self, indices) -> str:
+        """Generate appropriate error message based on number of exceptions."""
+        msg = f"Exceptions were raised in {len(indices)} interviews.\n"
+        if len(indices) > 5:
+            msg += f"Exceptions were raised in the following interviews: {indices}.\n"
+        return msg
+
+    def handle_exceptions(self) -> None:
+        """Handle exceptions by printing messages and generating reports as needed."""
+        if not (
+            self.results.has_unfixed_exceptions and self.parameters.print_exceptions
+        ):
+            return
+
+        # Print error message
+        error_msg = self._generate_error_message(self.results.task_history.indices)
+        print(error_msg, file=sys.stderr)
+
+        # Generate HTML report
+        filepath = self.results.task_history.html(
+            cta="Open report to see details.",
+            open_in_browser=self.open_in_browser,
+            return_link=True,
+        )
+
+        # Handle remote logging if enabled
+        if self.remote_logging:
+            filestore = HTMLFileStore(filepath)
+            coop_details = filestore.push(description="Error report")
+            print(coop_details)
+
+        print("Also see: https://docs.expectedparrot.com/en/latest/exceptions.html")