edsl 0.1.30.dev4__py3-none-any.whl → 0.1.31__py3-none-any.whl

edsl/jobs/Jobs.py

@@ -3,9 +3,7 @@ from __future__ import annotations
 import warnings
 from itertools import product
 from typing import Optional, Union, Sequence, Generator
-
 from edsl.Base import Base
-
 from edsl.exceptions import MissingAPIKeyError
 from edsl.jobs.buckets.BucketCollection import BucketCollection
 from edsl.jobs.interviews.Interview import Interview
@@ -321,7 +319,11 @@ class Jobs(Base):
         self.scenarios = self.scenarios or [Scenario()]
         for agent, scenario, model in product(self.agents, self.scenarios, self.models):
             yield Interview(
-                survey=self.survey, agent=agent, scenario=scenario, model=model
+                survey=self.survey,
+                agent=agent,
+                scenario=scenario,
+                model=model,
+                skip_retry=self.skip_retry,
             )
 
     def create_bucket_collection(self) -> BucketCollection:
@@ -411,6 +413,12 @@ class Jobs(Base):
                 if warn:
                     warnings.warn(message)
 
+    @property
+    def skip_retry(self):
+        if not hasattr(self, "_skip_retry"):
+            return False
+        return self._skip_retry
+
     def run(
         self,
         n: int = 1,
@@ -425,6 +433,7 @@ class Jobs(Base):
         print_exceptions=True,
         remote_cache_description: Optional[str] = None,
         remote_inference_description: Optional[str] = None,
+        skip_retry: bool = False,
     ) -> Results:
         """
         Runs the Job: conducts Interviews and returns their results.
@@ -443,6 +452,7 @@ class Jobs(Base):
         from edsl.coop.coop import Coop
 
         self._check_parameters()
+        self._skip_retry = skip_retry
 
         if batch_mode is not None:
             raise NotImplementedError(
@@ -461,12 +471,11 @@ class Jobs(Base):
             remote_inference = False
 
         if remote_inference:
-            from edsl.agents.Agent import Agent
-            from edsl.language_models.registry import Model
-            from edsl.results.Result import Result
-            from edsl.results.Results import Results
-            from edsl.scenarios.Scenario import Scenario
-            from edsl.surveys.Survey import Survey
+            import time
+            from datetime import datetime
+            from edsl.config import CONFIG
+
+            expected_parrot_url = CONFIG.get("EXPECTED_PARROT_URL")
 
             self._output("Remote inference activated. Sending job to server...")
             if remote_cache:
@@ -474,33 +483,60 @@ class Jobs(Base):
                     "Remote caching activated. The remote cache will be used for this job."
                 )
 
-            remote_job_data = coop.remote_inference_create(
+            remote_job_creation_data = coop.remote_inference_create(
                 self,
                 description=remote_inference_description,
                 status="queued",
+                iterations=n,
             )
-            self._output("Job sent!")
-            # Create mock results object to store job data
-            results = Results(
-                survey=Survey(),
-                data=[
-                    Result(
-                        agent=Agent.example(),
-                        scenario=Scenario.example(),
-                        model=Model(),
-                        iteration=1,
-                        answer={"info": "Remote job details"},
+            time_queued = datetime.now().strftime("%m/%d/%Y %I:%M:%S %p")
+            job_uuid = remote_job_creation_data.get("uuid")
+            print(f"Remote inference started (Job uuid={job_uuid}).")
+            # print(f"Job queued at {time_queued}.")
+            job_in_queue = True
+            while job_in_queue:
+                remote_job_data = coop.remote_inference_get(job_uuid)
+                status = remote_job_data.get("status")
+                if status == "cancelled":
+                    print("\r" + " " * 80 + "\r", end="")
+                    print("Job cancelled by the user.")
+                    print(
+                        f"See {expected_parrot_url}/home/remote-inference for more details."
                     )
-                ],
-            )
-            results.add_columns_from_dict([remote_job_data])
-            if self.verbose:
-                results.select(["info", "uuid", "status", "version"]).print(
-                    format="rich"
-                )
-            return results
+                    return None
+                elif status == "failed":
+                    print("\r" + " " * 80 + "\r", end="")
+                    print("Job failed.")
+                    print(
+                        f"See {expected_parrot_url}/home/remote-inference for more details."
+                    )
+                    return None
+                elif status == "completed":
+                    results_uuid = remote_job_data.get("results_uuid")
+                    results = coop.get(results_uuid, expected_object_type="results")
+                    print("\r" + " " * 80 + "\r", end="")
+                    print(
+                        f"Job completed and Results stored on Coop (Results uuid={results_uuid})."
+                    )
+                    return results
+                else:
+                    duration = 10 if len(self) < 10 else 60
+                    time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
+                    frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
+                    start_time = time.time()
+                    i = 0
+                    while time.time() - start_time < duration:
+                        print(
+                            f"\r{frames[i % len(frames)]} Job status: {status} - last update: {time_checked}",
+                            end="",
+                            flush=True,
+                        )
+                        time.sleep(0.1)
+                        i += 1
         else:
             if check_api_keys:
+                from edsl import Model
+
                 for model in self.models + [Model()]:
                     if not model.has_valid_api_key():
                         raise MissingAPIKeyError(
@@ -606,9 +642,9 @@ class Jobs(Base):
         results = JobsRunnerAsyncio(self).run(*args, **kwargs)
         return results
 
-    async def run_async(self, cache=None, **kwargs):
+    async def run_async(self, cache=None, n=1, **kwargs):
         """Run the job asynchronously."""
-        results = await JobsRunnerAsyncio(self).run_async(cache=cache, **kwargs)
+        results = await JobsRunnerAsyncio(self).run_async(cache=cache, n=n, **kwargs)
         return results
 
     def all_question_parameters(self):
@@ -688,7 +724,10 @@ class Jobs(Base):
     #######################
     @classmethod
     def example(
-        cls, throw_exception_probability: int = 0, randomize: bool = False
+        cls,
+        throw_exception_probability: int = 0,
+        randomize: bool = False,
+        test_model=False,
     ) -> Jobs:
         """Return an example Jobs instance.
 
@@ -706,6 +745,11 @@ class Jobs(Base):
 
         addition = "" if not randomize else str(uuid4())
 
+        if test_model:
+            from edsl.language_models import LanguageModel
+
+            m = LanguageModel.example(test_model=True)
+
         # (status, question, period)
         agent_answers = {
             ("Joyful", "how_feeling", "morning"): "OK",
@@ -753,7 +797,10 @@ class Jobs(Base):
                 Scenario({"period": "afternoon"}),
             ]
         )
-        job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
+        if test_model:
+            job = base_survey.by(m).by(scenario_list).by(joy_agent, sad_agent)
+        else:
+            job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
 
         return job
 

edsl/jobs/buckets/TokenBucket.py

@@ -30,6 +30,7 @@ class TokenBucket:
         if self.turbo_mode:
             pass
         else:
+            # pass
             self.turbo_mode = True
             self.capacity = float("inf")
             self.refill_rate = float("inf")
@@ -72,7 +73,17 @@ class TokenBucket:
         self.log.append((time.monotonic(), self.tokens))
 
     def refill(self) -> None:
-        """Refill the bucket with new tokens based on elapsed time."""
+        """Refill the bucket with new tokens based on elapsed time.
+
+
+
+        >>> bucket = TokenBucket(bucket_name="test", bucket_type="test", capacity=10, refill_rate=1)
+        >>> bucket.tokens = 0
+        >>> bucket.refill()
+        >>> bucket.tokens > 0
+        True
+
+        """
         now = time.monotonic()
         elapsed = now - self.last_refill
         refill_amount = elapsed * self.refill_rate
@@ -89,7 +100,9 @@ class TokenBucket:
         available_tokens = min(self.capacity, self.tokens + refill_amount)
         return max(0, requested_tokens - available_tokens) / self.refill_rate
 
-    async def get_tokens(self, amount: Union[int, float] = 1) -> None:
+    async def get_tokens(
+        self, amount: Union[int, float] = 1, cheat_bucket_capacity=True
+    ) -> None:
         """Wait for the specified number of tokens to become available.
 
 
@@ -105,14 +118,20 @@ class TokenBucket:
         True
 
         >>> bucket = TokenBucket(bucket_name="test", bucket_type="test", capacity=10, refill_rate=1)
-        >>> asyncio.run(bucket.get_tokens(11))
+        >>> asyncio.run(bucket.get_tokens(11, cheat_bucket_capacity=False))
         Traceback (most recent call last):
         ...
         ValueError: Requested amount exceeds bucket capacity. Bucket capacity: 10, requested amount: 11. As the bucket never overflows, the requested amount will never be available.
+        >>> asyncio.run(bucket.get_tokens(11, cheat_bucket_capacity=True))
         """
         if amount > self.capacity:
-            msg = f"Requested amount exceeds bucket capacity. Bucket capacity: {self.capacity}, requested amount: {amount}. As the bucket never overflows, the requested amount will never be available."
-            raise ValueError(msg)
+            if not cheat_bucket_capacity:
+                msg = f"Requested amount exceeds bucket capacity. Bucket capacity: {self.capacity}, requested amount: {amount}. As the bucket never overflows, the requested amount will never be available."
+                raise ValueError(msg)
+            else:
+                self.tokens = 0  # clear the bucket but let it go through
+                return
+
         while self.tokens < amount:
             self.refill()
             await asyncio.sleep(0.01)  # Sleep briefly to prevent busy waiting

edsl/jobs/interviews/Interview.py

@@ -14,12 +14,18 @@ from edsl.jobs.tasks.TaskCreators import TaskCreators
 from edsl.jobs.interviews.InterviewStatusLog import InterviewStatusLog
 from edsl.jobs.interviews.interview_exception_tracking import (
     InterviewExceptionCollection,
-    InterviewExceptionEntry,
 )
+from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.interviews.InterviewTaskBuildingMixin import InterviewTaskBuildingMixin
 from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
 
+import asyncio
+
+
+def run_async(coro):
+    return asyncio.run(coro)
+
 
 class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
     """
@@ -36,8 +42,9 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         model: Type["LanguageModel"],
         debug: Optional[bool] = False,
         iteration: int = 0,
-        cache: "Cache" = None,
-        sidecar_model: "LanguageModel" = None,
+        cache: Optional["Cache"] = None,
+        sidecar_model: Optional["LanguageModel"] = None,
+        skip_retry=False,
     ):
         """Initialize the Interview instance.
 
@@ -45,6 +52,24 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         :param survey: the survey being administered to the agent.
         :param scenario: the scenario that populates the survey questions.
         :param model: the language model used to answer the questions.
+        :param debug: if True, run without calls to the language model.
+        :param iteration: the iteration number of the interview.
+        :param cache: the cache used to store the answers.
+        :param sidecar_model: a sidecar model used to answer questions.
+
+        >>> i = Interview.example()
+        >>> i.task_creators
+        {}
+
+        >>> i.exceptions
+        {}
+
+        >>> _ = asyncio.run(i.async_conduct_interview())
+        >>> i.task_status_logs['q0']
+        [{'log_time': ..., 'value': <TaskStatus.NOT_STARTED: 1>}, {'log_time': ..., 'value': <TaskStatus.WAITING_FOR_DEPENDENCIES: 2>}, {'log_time': ..., 'value': <TaskStatus.API_CALL_IN_PROGRESS: 7>}, {'log_time': ..., 'value': <TaskStatus.SUCCESS: 8>}]
+
+        >>> i.to_index
+        {'q0': 0, 'q1': 1, 'q2': 2}
 
         """
         self.agent = agent
@@ -63,27 +88,54 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         self.task_creators = TaskCreators()  # tracks the task creators
         self.exceptions = InterviewExceptionCollection()
         self._task_status_log_dict = InterviewStatusLog()
+        self.skip_retry = skip_retry
 
-        # dictionary mapping question names to their index in the survey."""
+        # dictionary mapping question names to their index in the survey.
         self.to_index = {
             question_name: index
             for index, question_name in enumerate(self.survey.question_names)
         }
 
+    def _to_dict(self, include_exceptions=False) -> dict[str, Any]:
+        """Return a dictionary representation of the Interview instance.
+        This is just for hashing purposes.
+
+        >>> i = Interview.example()
+        >>> hash(i)
+        1646262796627658719
+        """
+        d = {
+            "agent": self.agent._to_dict(),
+            "survey": self.survey._to_dict(),
+            "scenario": self.scenario._to_dict(),
+            "model": self.model._to_dict(),
+            "iteration": self.iteration,
+            "exceptions": {},
+        }
+        if include_exceptions:
+            d["exceptions"] = self.exceptions.to_dict()
+
+    def __hash__(self) -> int:
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self._to_dict())
+
     async def async_conduct_interview(
         self,
         *,
         model_buckets: ModelBuckets = None,
         debug: bool = False,
         stop_on_exception: bool = False,
-        sidecar_model: Optional[LanguageModel] = None,
+        sidecar_model: Optional["LanguageModel"] = None,
     ) -> tuple["Answers", List[dict[str, Any]]]:
         """
         Conduct an Interview asynchronously.
+        It returns a tuple with the answers and a list of valid results.
 
         :param model_buckets: a dictionary of token buckets for the model.
         :param debug: run without calls to LLM.
         :param stop_on_exception: if True, stops the interview if an exception is raised.
+        :param sidecar_model: a sidecar model used to answer questions.
 
         Example usage:
 
@@ -91,17 +143,36 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         >>> result, _ = asyncio.run(i.async_conduct_interview())
         >>> result['q0']
         'yes'
+
+        >>> i = Interview.example(throw_exception = True)
+        >>> result, _ = asyncio.run(i.async_conduct_interview())
+        Attempt 1 failed with exception:This is a test error now waiting 1.00 seconds before retrying.Parameters: start=1.0, max=60.0, max_attempts=5.
+        <BLANKLINE>
+        <BLANKLINE>
+        Attempt 2 failed with exception:This is a test error now waiting 2.00 seconds before retrying.Parameters: start=1.0, max=60.0, max_attempts=5.
+        <BLANKLINE>
+        <BLANKLINE>
+        Attempt 3 failed with exception:This is a test error now waiting 4.00 seconds before retrying.Parameters: start=1.0, max=60.0, max_attempts=5.
+        <BLANKLINE>
+        <BLANKLINE>
+        Attempt 4 failed with exception:This is a test error now waiting 8.00 seconds before retrying.Parameters: start=1.0, max=60.0, max_attempts=5.
+        <BLANKLINE>
+        <BLANKLINE>
+
+        >>> i.exceptions
+        {'q0': ...
+        >>> i = Interview.example()
+        >>> result, _ = asyncio.run(i.async_conduct_interview(stop_on_exception = True))
+        Traceback (most recent call last):
+        ...
+        asyncio.exceptions.CancelledError
         """
         self.sidecar_model = sidecar_model
 
         # if no model bucket is passed, create an 'infinity' bucket with no rate limits
-        # print("model_buckets", model_buckets)
         if model_buckets is None or hasattr(self.agent, "answer_question_directly"):
             model_buckets = ModelBuckets.infinity_bucket()
 
-        # FOR TESTING
-        # model_buckets = ModelBuckets.infinity_bucket()
-
         ## build the tasks using the InterviewTaskBuildingMixin
         ## This is the key part---it creates a task for each question,
         ## with dependencies on the questions that must be answered before this one can be answered.
@@ -123,6 +194,14 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         It iterates through the tasks and invigilators, and yields the results of the tasks that are done.
         If a task is not done, it raises a ValueError.
         If an exception is raised in the task, it records the exception in the Interview instance except if the task was cancelled, which is expected behavior.
+
+        >>> i = Interview.example()
+        >>> result, _ = asyncio.run(i.async_conduct_interview())
+        >>> results = list(i._extract_valid_results())
+        >>> len(results) == len(i.survey)
+        True
+        >>> type(results[0])
+        <class 'edsl.data_transfer_models.AgentResponseDict'>
         """
         assert len(self.tasks) == len(self.invigilators)
 
@@ -140,12 +219,19 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
             yield result
 
     def _record_exception(self, task, exception: Exception) -> None:
-        """Record an exception in the Interview instance."""
-        exception_entry = InterviewExceptionEntry(
-            exception=repr(exception),
-            time=time.time(),
-            traceback=traceback.format_exc(),
-        )
+        """Record an exception in the Interview instance.
+
+        It records the exception in the Interview instance, with the task name and the exception entry.
+
+        >>> i = Interview.example()
+        >>> result, _ = asyncio.run(i.async_conduct_interview())
+        >>> i.exceptions
+        {}
+        >>> i._record_exception(i.tasks[0], Exception("An exception occurred."))
+        >>> i.exceptions
+        {'q0': ...
+        """
+        exception_entry = InterviewExceptionEntry(exception)
         self.exceptions.add(task.get_name(), exception_entry)
 
     @property
@@ -156,6 +242,10 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         It is used to determine the order in which questions should be answered.
         This reflects both agent 'memory' considerations and 'skip' logic.
         The 'textify' parameter is set to True, so that the question names are returned as strings rather than integer indices.
+
+        >>> i = Interview.example()
+        >>> i.dag == {'q2': {'q0'}, 'q1': {'q0'}}
+        True
         """
         return self.survey.dag(textify=True)
 
@@ -166,8 +256,15 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         """Return a string representation of the Interview instance."""
         return f"Interview(agent = {repr(self.agent)}, survey = {repr(self.survey)}, scenario = {repr(self.scenario)}, model = {repr(self.model)})"
 
-    def duplicate(self, iteration: int, cache: Cache) -> Interview:
-        """Duplicate the interview, but with a new iteration number and cache."""
+    def duplicate(self, iteration: int, cache: "Cache") -> Interview:
+        """Duplicate the interview, but with a new iteration number and cache.
+
+        >>> i = Interview.example()
+        >>> i2 = i.duplicate(1, None)
+        >>> i.iteration + 1 == i2.iteration
+        True
+
+        """
         return Interview(
             agent=self.agent,
             survey=self.survey,
@@ -175,10 +272,11 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
             model=self.model,
             iteration=iteration,
             cache=cache,
+            skip_retry=self.skip_retry,
         )
 
     @classmethod
-    def example(self):
+    def example(self, throw_exception: bool = False) -> Interview:
         """Return an example Interview instance."""
         from edsl.agents import Agent
         from edsl.surveys import Survey
@@ -193,66 +291,15 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         survey = Survey.example()
         scenario = Scenario.example()
         model = LanguageModel.example()
+        if throw_exception:
+            model = LanguageModel.example(test_model=True, throw_exception=True)
+            agent = Agent.example()
+            return Interview(agent=agent, survey=survey, scenario=scenario, model=model)
         return Interview(agent=agent, survey=survey, scenario=scenario, model=model)
 
 
 if __name__ == "__main__":
     import doctest
 
-    doctest.testmod()
-    # from edsl import Model
-    # from edsl.agents import Agent
-    # from edsl.surveys import Survey
-    # from edsl.scenarios import Scenario
-    # from edsl.questions import QuestionMultipleChoice
-
-    # # from edsl.jobs.Interview import Interview
-
-    # #  a survey with skip logic
-    # q0 = QuestionMultipleChoice(
-    #     question_text="Do you like school?",
-    #     question_options=["yes", "no"],
-    #     question_name="q0",
-    # )
-    # q1 = QuestionMultipleChoice(
-    #     question_text="Why not?",
-    #     question_options=["killer bees in cafeteria", "other"],
-    #     question_name="q1",
-    # )
-    # q2 = QuestionMultipleChoice(
-    #     question_text="Why?",
-    #     question_options=["**lack*** of killer bees in cafeteria", "other"],
-    #     question_name="q2",
-    # )
-    # s = Survey(questions=[q0, q1, q2])
-    # s = s.add_rule(q0, "q0 == 'yes'", q2)
-
-    # # create an interview
-    # a = Agent(traits=None)
-
-    # def direct_question_answering_method(self, question, scenario):
-    #     """Answer a question directly."""
-    #     raise Exception("Error!")
-    #     # return "yes"
-
-    # a.add_direct_question_answering_method(direct_question_answering_method)
-    # scenario = Scenario()
-    # m = Model()
-    # I = Interview(agent=a, survey=s, scenario=scenario, model=m)
-
-    # result = asyncio.run(I.async_conduct_interview())
-    # # # conduct five interviews
-    # # for _ in range(5):
-    # #     I.conduct_interview(debug=True)
-
-    # # # replace missing answers
-    # # I
-    # # repr(I)
-    # # eval(repr(I))
-    # # print(I.task_status_logs.status_matrix(20))
-    # status_matrix = I.task_status_logs.status_matrix(20)
-    # numerical_matrix = I.task_status_logs.numerical_matrix(20)
-    # I.task_status_logs.visualize()
-
-    # I.exceptions.print()
-    # I.exceptions.ascii_table()
+    # add ellipsis
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -0,0 +1,101 @@
+import traceback
+import datetime
+import time
+from collections import UserDict
+
+# traceback=traceback.format_exc(),
+# traceback = frame_summary_to_dict(traceback.extract_tb(e.__traceback__))
+# traceback = [frame_summary_to_dict(f) for f in traceback.extract_tb(e.__traceback__)]
+
+
+class InterviewExceptionEntry:
+    """Class to record an exception that occurred during the interview.
+
+    >>> entry = InterviewExceptionEntry.example()
+    >>> entry.to_dict()['exception']
+    "ValueError('An error occurred.')"
+    """
+
+    def __init__(self, exception: Exception, traceback_format="html"):
+        self.time = datetime.datetime.now().isoformat()
+        self.exception = exception
+        self.traceback_format = traceback_format
+
+    def __getitem__(self, key):
+        # Support dict-like access obj['a']
+        return str(getattr(self, key))
+
+    @classmethod
+    def example(cls):
+        try:
+            raise ValueError("An error occurred.")
+        except Exception as e:
+            entry = InterviewExceptionEntry(e)
+        return entry
+
+    @property
+    def traceback(self):
+        """Return the exception as HTML."""
+        if self.traceback_format == "html":
+            return self.html_traceback
+        else:
+            return self.text_traceback
+
+    @property
+    def text_traceback(self):
+        """
+        >>> entry = InterviewExceptionEntry.example()
+        >>> entry.text_traceback
+        'Traceback (most recent call last):...'
+        """
+        e = self.exception
+        tb_str = "".join(traceback.format_exception(type(e), e, e.__traceback__))
+        return tb_str
+
+    @property
+    def html_traceback(self):
+        from rich.console import Console
+        from rich.table import Table
+        from rich.traceback import Traceback
+
+        from io import StringIO
+
+        html_output = StringIO()
+
+        console = Console(file=html_output, record=True)
+
+        tb = Traceback.from_exception(
+            type(self.exception),
+            self.exception,
+            self.exception.__traceback__,
+            show_locals=True,
+        )
+        console.print(tb)
+        return html_output.getvalue()
+
+    def to_dict(self) -> dict:
+        """Return the exception as a dictionary.
+
+        >>> entry = InterviewExceptionEntry.example()
+        >>> entry.to_dict()['exception']
+        "ValueError('An error occurred.')"
+
+        """
+        return {
+            "exception": repr(self.exception),
+            "time": self.time,
+            "traceback": self.traceback,
+        }
+
+    def push(self):
+        from edsl import Coop
+
+        coop = Coop()
+        results = coop.error_create(self.to_dict())
+        return results
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)