edsl 0.1.29.dev6__py3-none-any.whl → 0.1.30__py3-none-any.whl

edsl/jobs/Jobs.py

@@ -1,30 +1,15 @@
 # """The Jobs class is a collection of agents, scenarios and models and one survey."""
 from __future__ import annotations
-import os
 import warnings
 from itertools import product
 from typing import Optional, Union, Sequence, Generator
-from edsl import Model
-from edsl.agents import Agent
-from edsl.agents.AgentList import AgentList
+
 from edsl.Base import Base
-from edsl.data.Cache import Cache
-from edsl.data.CacheHandler import CacheHandler
-from edsl.results.Dataset import Dataset
 
-from edsl.exceptions.jobs import MissingRemoteInferenceError
 from edsl.exceptions import MissingAPIKeyError
 from edsl.jobs.buckets.BucketCollection import BucketCollection
 from edsl.jobs.interviews.Interview import Interview
-from edsl.language_models import LanguageModel
-from edsl.results import Results
-from edsl.scenarios import Scenario
-from edsl import ScenarioList
-from edsl.surveys import Survey
 from edsl.jobs.runners.JobsRunnerAsyncio import JobsRunnerAsyncio
-
-from edsl.language_models.ModelList import ModelList
-
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
 
@@ -37,10 +22,10 @@ class Jobs(Base):
 
     def __init__(
         self,
-        survey: Survey,
-        agents: Optional[list[Agent]] = None,
-        models: Optional[list[LanguageModel]] = None,
-        scenarios: Optional[list[Scenario]] = None,
+        survey: "Survey",
+        agents: Optional[list["Agent"]] = None,
+        models: Optional[list["LanguageModel"]] = None,
+        scenarios: Optional[list["Scenario"]] = None,
     ):
         """Initialize a Jobs instance.
 
@@ -50,8 +35,8 @@ class Jobs(Base):
         :param scenarios: a list of scenarios
         """
         self.survey = survey
-        self.agents: AgentList = agents
-        self.scenarios: ScenarioList = scenarios
+        self.agents: "AgentList" = agents
+        self.scenarios: "ScenarioList" = scenarios
         self.models = models
 
         self.__bucket_collection = None
@@ -62,6 +47,8 @@ class Jobs(Base):
 
     @models.setter
     def models(self, value):
+        from edsl import ModelList
+
         if value:
             if not isinstance(value, ModelList):
                 self._models = ModelList(value)
@@ -76,6 +63,8 @@ class Jobs(Base):
 
     @agents.setter
     def agents(self, value):
+        from edsl import AgentList
+
         if value:
             if not isinstance(value, AgentList):
                 self._agents = AgentList(value)
@@ -90,6 +79,8 @@ class Jobs(Base):
 
     @scenarios.setter
     def scenarios(self, value):
+        from edsl import ScenarioList
+
         if value:
             if not isinstance(value, ScenarioList):
                 self._scenarios = ScenarioList(value)
@@ -101,10 +92,10 @@ class Jobs(Base):
     def by(
         self,
         *args: Union[
-            Agent,
-            Scenario,
-            LanguageModel,
-            Sequence[Union[Agent, Scenario, LanguageModel]],
+            "Agent",
+            "Scenario",
+            "LanguageModel",
+            Sequence[Union["Agent", "Scenario", "LanguageModel"]],
         ],
     ) -> Jobs:
         """
@@ -144,7 +135,7 @@ class Jobs(Base):
         setattr(self, objects_key, new_objects)  # update the job
         return self
 
-    def prompts(self) -> Dataset:
+    def prompts(self) -> "Dataset":
         """Return a Dataset of prompts that will be used.
 
 
@@ -160,6 +151,7 @@ class Jobs(Base):
         user_prompts = []
         system_prompts = []
         scenario_indices = []
+        from edsl.results.Dataset import Dataset
 
         for interview_index, interview in enumerate(interviews):
             invigilators = list(interview._build_invigilators(debug=False))
@@ -182,7 +174,10 @@ class Jobs(Base):
 
     @staticmethod
     def _get_container_class(object):
-        from edsl import AgentList
+        from edsl.agents.AgentList import AgentList
+        from edsl.agents.Agent import Agent
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.scenarios.ScenarioList import ScenarioList
 
         if isinstance(object, Agent):
             return AgentList
@@ -218,6 +213,10 @@ class Jobs(Base):
     def _get_current_objects_of_this_type(
         self, object: Union[Agent, Scenario, LanguageModel]
     ) -> tuple[list, str]:
+        from edsl.agents.Agent import Agent
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.language_models.LanguageModel import LanguageModel
+
         """Return the current objects of the same type as the first argument.
 
         >>> from edsl.jobs import Jobs
@@ -246,6 +245,9 @@ class Jobs(Base):
     @staticmethod
     def _get_empty_container_object(object):
         from edsl import AgentList
+        from edsl import Agent
+        from edsl import Scenario
+        from edsl import ScenarioList
 
         if isinstance(object, Agent):
             return AgentList([])
@@ -310,6 +312,10 @@ class Jobs(Base):
         with us filling in defaults.
         """
         # if no agents, models, or scenarios are set, set them to defaults
+        from edsl.agents.Agent import Agent
+        from edsl.language_models.registry import Model
+        from edsl.scenarios.Scenario import Scenario
+
         self.agents = self.agents or [Agent()]
         self.models = self.models or [Model()]
         self.scenarios = self.scenarios or [Scenario()]
@@ -325,6 +331,7 @@ class Jobs(Base):
         These buckets are used to track API calls and token usage.
 
         >>> from edsl.jobs import Jobs
+        >>> from edsl import Model
         >>> j = Jobs.example().by(Model(temperature = 1), Model(temperature = 0.5))
         >>> bc = j.create_bucket_collection()
         >>> bc
@@ -368,6 +375,8 @@ class Jobs(Base):
         """Check if the parameters in the survey and scenarios are consistent.
 
         >>> from edsl import QuestionFreeText
+        >>> from edsl import Survey
+        >>> from edsl import Scenario
         >>> q = QuestionFreeText(question_text = "{{poo}}", question_name = "ugly_question")
         >>> j = Jobs(survey = Survey(questions=[q]))
         >>> with warnings.catch_warnings(record=True) as w:
@@ -452,6 +461,13 @@ class Jobs(Base):
             remote_inference = False
 
         if remote_inference:
+            from edsl.agents.Agent import Agent
+            from edsl.language_models.registry import Model
+            from edsl.results.Result import Result
+            from edsl.results.Results import Results
+            from edsl.scenarios.Scenario import Scenario
+            from edsl.surveys.Survey import Survey
+
             self._output("Remote inference activated. Sending job to server...")
             if remote_cache:
                 self._output(
@@ -464,8 +480,25 @@ class Jobs(Base):
                 status="queued",
             )
             self._output("Job sent!")
-            self._output(remote_job_data)
-            return remote_job_data
+            # Create mock results object to store job data
+            results = Results(
+                survey=Survey(),
+                data=[
+                    Result(
+                        agent=Agent.example(),
+                        scenario=Scenario.example(),
+                        model=Model(),
+                        iteration=1,
+                        answer={"info": "Remote job details"},
+                    )
+                ],
+            )
+            results.add_columns_from_dict([remote_job_data])
+            if self.verbose:
+                results.select(["info", "uuid", "status", "version"]).print(
+                    format="rich"
+                )
+            return results
         else:
             if check_api_keys:
                 for model in self.models + [Model()]:
@@ -477,8 +510,12 @@ class Jobs(Base):
 
         # handle cache
         if cache is None:
+            from edsl.data.CacheHandler import CacheHandler
+
             cache = CacheHandler().get_cache()
         if cache is False:
+            from edsl.data.Cache import Cache
+
             cache = Cache()
 
         if not remote_cache:
@@ -630,6 +667,11 @@ class Jobs(Base):
     @remove_edsl_version
     def from_dict(cls, data: dict) -> Jobs:
         """Creates a Jobs instance from a dictionary."""
+        from edsl import Survey
+        from edsl.agents.Agent import Agent
+        from edsl.language_models.LanguageModel import LanguageModel
+        from edsl.scenarios.Scenario import Scenario
+
         return cls(
             survey=Survey.from_dict(data["survey"]),
             agents=[Agent.from_dict(agent) for agent in data["agents"]],
@@ -645,7 +687,9 @@ class Jobs(Base):
     # Example methods
     #######################
     @classmethod
-    def example(cls, throw_exception_probability=0) -> Jobs:
+    def example(
+        cls, throw_exception_probability: int = 0, randomize: bool = False
+    ) -> Jobs:
         """Return an example Jobs instance.
 
         :param throw_exception_probability: the probability that an exception will be thrown when answering a question. This is useful for testing error handling.
@@ -655,8 +699,12 @@ class Jobs(Base):
 
         """
         import random
+        from uuid import uuid4
         from edsl.questions import QuestionMultipleChoice
-        from edsl import Agent
+        from edsl.agents.Agent import Agent
+        from edsl.scenarios.Scenario import Scenario
+
+        addition = "" if not randomize else str(uuid4())
 
         # (status, question, period)
         agent_answers = {
@@ -695,10 +743,15 @@ class Jobs(Base):
             question_options=["Good", "Great", "OK", "Terrible"],
             question_name="how_feeling_yesterday",
         )
+        from edsl import Survey, ScenarioList
+
         base_survey = Survey(questions=[q1, q2])
 
         scenario_list = ScenarioList(
-            [Scenario({"period": "morning"}), Scenario({"period": "afternoon"})]
+            [
+                Scenario({"period": f"morning{addition}"}),
+                Scenario({"period": "afternoon"}),
+            ]
         )
         job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
 

edsl/jobs/buckets/ModelBuckets.py

@@ -1,4 +1,4 @@
-from edsl.jobs.buckets.TokenBucket import TokenBucket
+# from edsl.jobs.buckets.TokenBucket import TokenBucket
 
 
 class ModelBuckets:
@@ -8,7 +8,7 @@ class ModelBuckets:
     A request is one call to the service. The number of tokens required for a request depends on parameters.
     """
 
-    def __init__(self, requests_bucket: TokenBucket, tokens_bucket: TokenBucket):
+    def __init__(self, requests_bucket: "TokenBucket", tokens_bucket: "TokenBucket"):
         """Initialize the model buckets.
 
         The requests bucket captures requests per unit of time.
@@ -25,9 +25,21 @@ class ModelBuckets:
             tokens_bucket=self.tokens_bucket + other.tokens_bucket,
         )
 
+    def turbo_mode_on(self):
+        """Set the refill rate to infinity for both buckets."""
+        self.requests_bucket.turbo_mode_on()
+        self.tokens_bucket.turbo_mode_on()
+
+    def turbo_mode_off(self):
+        """Restore the refill rate to its original value for both buckets."""
+        self.requests_bucket.turbo_mode_off()
+        self.tokens_bucket.turbo_mode_off()
+
     @classmethod
     def infinity_bucket(cls, model_name: str = "not_specified") -> "ModelBuckets":
         """Create a bucket with infinite capacity and refill rate."""
+        from edsl.jobs.buckets.TokenBucket import TokenBucket
+
         return cls(
             requests_bucket=TokenBucket(
                 bucket_name=model_name,

edsl/jobs/buckets/TokenBucket.py

@@ -1,8 +1,6 @@
 from typing import Union, List, Any
 import asyncio
 import time
-from collections import UserDict
-from matplotlib import pyplot as plt
 
 
 class TokenBucket:
@@ -19,11 +17,29 @@ class TokenBucket:
         self.bucket_name = bucket_name
         self.bucket_type = bucket_type
         self.capacity = capacity  # Maximum number of tokens
+        self._old_capacity = capacity
         self.tokens = capacity  # Current number of available tokens
         self.refill_rate = refill_rate  # Rate at which tokens are refilled
+        self._old_refill_rate = refill_rate
         self.last_refill = time.monotonic()  # Last refill time
-
         self.log: List[Any] = []
+        self.turbo_mode = False
+
+    def turbo_mode_on(self):
+        """Set the refill rate to infinity."""
+        if self.turbo_mode:
+            pass
+        else:
+            # pass
+            self.turbo_mode = True
+            self.capacity = float("inf")
+            self.refill_rate = float("inf")
+
+    def turbo_mode_off(self):
+        """Restore the refill rate to its original value."""
+        self.turbo_mode = False
+        self.capacity = self._old_capacity
+        self.refill_rate = self._old_refill_rate
 
     def __add__(self, other) -> "TokenBucket":
         """Combine two token buckets.
@@ -57,7 +73,17 @@ class TokenBucket:
         self.log.append((time.monotonic(), self.tokens))
 
     def refill(self) -> None:
-        """Refill the bucket with new tokens based on elapsed time."""
+        """Refill the bucket with new tokens based on elapsed time.
+
+
+
+        >>> bucket = TokenBucket(bucket_name="test", bucket_type="test", capacity=10, refill_rate=1)
+        >>> bucket.tokens = 0
+        >>> bucket.refill()
+        >>> bucket.tokens > 0
+        True
+
+        """
         now = time.monotonic()
         elapsed = now - self.last_refill
         refill_amount = elapsed * self.refill_rate
@@ -100,7 +126,7 @@ class TokenBucket:
             raise ValueError(msg)
         while self.tokens < amount:
             self.refill()
-            await asyncio.sleep(0.1)  # Sleep briefly to prevent busy waiting
+            await asyncio.sleep(0.01)  # Sleep briefly to prevent busy waiting
         self.tokens -= amount
 
         now = time.monotonic()
@@ -114,6 +140,7 @@ class TokenBucket:
         times, tokens = zip(*self.get_log())
         start_time = times[0]
         times = [t - start_time for t in times]  # Normalize time to start from 0
+        from matplotlib import pyplot as plt
 
         plt.figure(figsize=(10, 6))
         plt.plot(times, tokens, label="Tokens Available")

edsl/jobs/interviews/Interview.py

@@ -6,15 +6,9 @@ import asyncio
 import time
 from typing import Any, Type, List, Generator, Optional
 
-from edsl.agents import Agent
-from edsl.language_models import LanguageModel
-from edsl.scenarios import Scenario
-from edsl.surveys import Survey
-
 from edsl.jobs.Answers import Answers
 from edsl.surveys.base import EndOfSurvey
 from edsl.jobs.buckets.ModelBuckets import ModelBuckets
-
 from edsl.jobs.tasks.TaskCreators import TaskCreators
 
 from edsl.jobs.interviews.InterviewStatusLog import InterviewStatusLog
@@ -26,6 +20,12 @@ from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.interviews.InterviewTaskBuildingMixin import InterviewTaskBuildingMixin
 from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
 
+import asyncio
+
+
+def run_async(coro):
+    return asyncio.run(coro)
+
 
 class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
     """
@@ -36,14 +36,14 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
 
     def __init__(
         self,
-        agent: Agent,
-        survey: Survey,
-        scenario: Scenario,
-        model: Type[LanguageModel],
-        debug: bool = False,
+        agent: "Agent",
+        survey: "Survey",
+        scenario: "Scenario",
+        model: Type["LanguageModel"],
+        debug: Optional[bool] = False,
         iteration: int = 0,
-        cache: "Cache" = None,
-        sidecar_model: LanguageModel = None,
+        cache: Optional["Cache"] = None,
+        sidecar_model: Optional["LanguageModel"] = None,
     ):
         """Initialize the Interview instance.
 
@@ -51,6 +51,24 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         :param survey: the survey being administered to the agent.
         :param scenario: the scenario that populates the survey questions.
         :param model: the language model used to answer the questions.
+        :param debug: if True, run without calls to the language model.
+        :param iteration: the iteration number of the interview.
+        :param cache: the cache used to store the answers.
+        :param sidecar_model: a sidecar model used to answer questions.
+
+        >>> i = Interview.example()
+        >>> i.task_creators
+        {}
+
+        >>> i.exceptions
+        {}
+
+        >>> _ = asyncio.run(i.async_conduct_interview())
+        >>> i.task_status_logs['q0']
+        [{'log_time': ..., 'value': <TaskStatus.NOT_STARTED: 1>}, {'log_time': ..., 'value': <TaskStatus.WAITING_FOR_DEPENDENCIES: 2>}, {'log_time': ..., 'value': <TaskStatus.API_CALL_IN_PROGRESS: 7>}, {'log_time': ..., 'value': <TaskStatus.SUCCESS: 8>}]
+
+        >>> i.to_index
+        {'q0': 0, 'q1': 1, 'q2': 2}
 
         """
         self.agent = agent
@@ -70,7 +88,7 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         self.exceptions = InterviewExceptionCollection()
         self._task_status_log_dict = InterviewStatusLog()
 
-        # dictionary mapping question names to their index in the survey."""
+        # dictionary mapping question names to their index in the survey.
         self.to_index = {
             question_name: index
             for index, question_name in enumerate(self.survey.question_names)
@@ -82,14 +100,16 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         model_buckets: ModelBuckets = None,
         debug: bool = False,
         stop_on_exception: bool = False,
-        sidecar_model: Optional[LanguageModel] = None,
+        sidecar_model: Optional["LanguageModel"] = None,
     ) -> tuple["Answers", List[dict[str, Any]]]:
         """
         Conduct an Interview asynchronously.
+        It returns a tuple with the answers and a list of valid results.
 
         :param model_buckets: a dictionary of token buckets for the model.
         :param debug: run without calls to LLM.
         :param stop_on_exception: if True, stops the interview if an exception is raised.
+        :param sidecar_model: a sidecar model used to answer questions.
 
         Example usage:
 
@@ -97,16 +117,37 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         >>> result, _ = asyncio.run(i.async_conduct_interview())
         >>> result['q0']
         'yes'
+
+        >>> i = Interview.example(throw_exception = True)
+        >>> result, _ = asyncio.run(i.async_conduct_interview())
+        Attempt 1 failed with exception:This is a test error now waiting 1.00 seconds before retrying.Parameters: start=1.0, max=60.0, max_attempts=5.
+        <BLANKLINE>
+        <BLANKLINE>
+        Attempt 2 failed with exception:This is a test error now waiting 2.00 seconds before retrying.Parameters: start=1.0, max=60.0, max_attempts=5.
+        <BLANKLINE>
+        <BLANKLINE>
+        Attempt 3 failed with exception:This is a test error now waiting 4.00 seconds before retrying.Parameters: start=1.0, max=60.0, max_attempts=5.
+        <BLANKLINE>
+        <BLANKLINE>
+        Attempt 4 failed with exception:This is a test error now waiting 8.00 seconds before retrying.Parameters: start=1.0, max=60.0, max_attempts=5.
+        <BLANKLINE>
+        <BLANKLINE>
+
+        >>> i.exceptions
+        {'q0': [{'exception': "Exception('This is a test error')", 'time': ..., 'traceback': ...
+
+        >>> i = Interview.example()
+        >>> result, _ = asyncio.run(i.async_conduct_interview(stop_on_exception = True))
+        Traceback (most recent call last):
+        ...
+        asyncio.exceptions.CancelledError
         """
         self.sidecar_model = sidecar_model
 
         # if no model bucket is passed, create an 'infinity' bucket with no rate limits
-        # print("model_buckets", model_buckets)
         if model_buckets is None or hasattr(self.agent, "answer_question_directly"):
             model_buckets = ModelBuckets.infinity_bucket()
 
-        # model_buckets = ModelBuckets.infinity_bucket()
-
         ## build the tasks using the InterviewTaskBuildingMixin
         ## This is the key part---it creates a task for each question,
         ## with dependencies on the questions that must be answered before this one can be answered.
@@ -128,6 +169,14 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         It iterates through the tasks and invigilators, and yields the results of the tasks that are done.
         If a task is not done, it raises a ValueError.
         If an exception is raised in the task, it records the exception in the Interview instance except if the task was cancelled, which is expected behavior.
+
+        >>> i = Interview.example()
+        >>> result, _ = asyncio.run(i.async_conduct_interview())
+        >>> results = list(i._extract_valid_results())
+        >>> len(results) == len(i.survey)
+        True
+        >>> type(results[0])
+        <class 'edsl.data_transfer_models.AgentResponseDict'>
         """
         assert len(self.tasks) == len(self.invigilators)
 
@@ -145,7 +194,18 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
             yield result
 
     def _record_exception(self, task, exception: Exception) -> None:
-        """Record an exception in the Interview instance."""
+        """Record an exception in the Interview instance.
+
+        It records the exception in the Interview instance, with the task name and the exception entry.
+
+        >>> i = Interview.example()
+        >>> result, _ = asyncio.run(i.async_conduct_interview())
+        >>> i.exceptions
+        {}
+        >>> i._record_exception(i.tasks[0], Exception("An exception occurred."))
+        >>> i.exceptions
+        {'q0': [{'exception': "Exception('An exception occurred.')", 'time': ..., 'traceback': 'NoneType: None\\n'}]}
+        """
         exception_entry = InterviewExceptionEntry(
             exception=repr(exception),
             time=time.time(),
@@ -161,6 +221,10 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         It is used to determine the order in which questions should be answered.
         This reflects both agent 'memory' considerations and 'skip' logic.
         The 'textify' parameter is set to True, so that the question names are returned as strings rather than integer indices.
+
+        >>> i = Interview.example()
+        >>> i.dag == {'q2': {'q0'}, 'q1': {'q0'}}
+        True
         """
         return self.survey.dag(textify=True)
 
@@ -171,8 +235,15 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         """Return a string representation of the Interview instance."""
         return f"Interview(agent = {repr(self.agent)}, survey = {repr(self.survey)}, scenario = {repr(self.scenario)}, model = {repr(self.model)})"
 
-    def duplicate(self, iteration: int, cache: Cache) -> Interview:
-        """Duplicate the interview, but with a new iteration number and cache."""
+    def duplicate(self, iteration: int, cache: "Cache") -> Interview:
+        """Duplicate the interview, but with a new iteration number and cache.
+
+        >>> i = Interview.example()
+        >>> i2 = i.duplicate(1, None)
+        >>> i.iteration + 1 == i2.iteration
+        True
+
+        """
         return Interview(
             agent=self.agent,
             survey=self.survey,
@@ -183,7 +254,7 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         )
 
     @classmethod
-    def example(self):
+    def example(self, throw_exception: bool = False) -> Interview:
         """Return an example Interview instance."""
         from edsl.agents import Agent
         from edsl.surveys import Survey
@@ -198,66 +269,15 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         survey = Survey.example()
         scenario = Scenario.example()
         model = LanguageModel.example()
+        if throw_exception:
+            model = LanguageModel.example(test_model=True, throw_exception=True)
+            agent = Agent.example()
+            return Interview(agent=agent, survey=survey, scenario=scenario, model=model)
         return Interview(agent=agent, survey=survey, scenario=scenario, model=model)
 
 
 if __name__ == "__main__":
     import doctest
 
-    doctest.testmod()
-    # from edsl import Model
-    # from edsl.agents import Agent
-    # from edsl.surveys import Survey
-    # from edsl.scenarios import Scenario
-    # from edsl.questions import QuestionMultipleChoice
-
-    # # from edsl.jobs.Interview import Interview
-
-    # #  a survey with skip logic
-    # q0 = QuestionMultipleChoice(
-    #     question_text="Do you like school?",
-    #     question_options=["yes", "no"],
-    #     question_name="q0",
-    # )
-    # q1 = QuestionMultipleChoice(
-    #     question_text="Why not?",
-    #     question_options=["killer bees in cafeteria", "other"],
-    #     question_name="q1",
-    # )
-    # q2 = QuestionMultipleChoice(
-    #     question_text="Why?",
-    #     question_options=["**lack*** of killer bees in cafeteria", "other"],
-    #     question_name="q2",
-    # )
-    # s = Survey(questions=[q0, q1, q2])
-    # s = s.add_rule(q0, "q0 == 'yes'", q2)
-
-    # # create an interview
-    # a = Agent(traits=None)
-
-    # def direct_question_answering_method(self, question, scenario):
-    #     """Answer a question directly."""
-    #     raise Exception("Error!")
-    #     # return "yes"
-
-    # a.add_direct_question_answering_method(direct_question_answering_method)
-    # scenario = Scenario()
-    # m = Model()
-    # I = Interview(agent=a, survey=s, scenario=scenario, model=m)
-
-    # result = asyncio.run(I.async_conduct_interview())
-    # # # conduct five interviews
-    # # for _ in range(5):
-    # #     I.conduct_interview(debug=True)
-
-    # # # replace missing answers
-    # # I
-    # # repr(I)
-    # # eval(repr(I))
-    # # print(I.task_status_logs.status_matrix(20))
-    # status_matrix = I.task_status_logs.status_matrix(20)
-    # numerical_matrix = I.task_status_logs.numerical_matrix(20)
-    # I.task_status_logs.visualize()
-
-    # I.exceptions.print()
-    # I.exceptions.ascii_table()
+    # add ellipsis
+    doctest.testmod(optionflags=doctest.ELLIPSIS)