edsl 0.1.30.dev5__py3-none-any.whl → 0.1.31__py3-none-any.whl

edsl/jobs/Jobs.py

@@ -3,9 +3,7 @@ from __future__ import annotations
 import warnings
 from itertools import product
 from typing import Optional, Union, Sequence, Generator
-
 from edsl.Base import Base
-
 from edsl.exceptions import MissingAPIKeyError
 from edsl.jobs.buckets.BucketCollection import BucketCollection
 from edsl.jobs.interviews.Interview import Interview
@@ -321,7 +319,11 @@ class Jobs(Base):
         self.scenarios = self.scenarios or [Scenario()]
         for agent, scenario, model in product(self.agents, self.scenarios, self.models):
             yield Interview(
-                survey=self.survey, agent=agent, scenario=scenario, model=model
+                survey=self.survey,
+                agent=agent,
+                scenario=scenario,
+                model=model,
+                skip_retry=self.skip_retry,
             )
 
     def create_bucket_collection(self) -> BucketCollection:
@@ -411,6 +413,12 @@ class Jobs(Base):
                 if warn:
                     warnings.warn(message)
 
+    @property
+    def skip_retry(self):
+        if not hasattr(self, "_skip_retry"):
+            return False
+        return self._skip_retry
+
     def run(
         self,
         n: int = 1,
@@ -425,6 +433,7 @@ class Jobs(Base):
         print_exceptions=True,
         remote_cache_description: Optional[str] = None,
         remote_inference_description: Optional[str] = None,
+        skip_retry: bool = False,
     ) -> Results:
         """
         Runs the Job: conducts Interviews and returns their results.
@@ -443,6 +452,7 @@ class Jobs(Base):
         from edsl.coop.coop import Coop
 
         self._check_parameters()
+        self._skip_retry = skip_retry
 
         if batch_mode is not None:
             raise NotImplementedError(
@@ -461,12 +471,11 @@ class Jobs(Base):
             remote_inference = False
 
         if remote_inference:
-            from edsl.agents.Agent import Agent
-            from edsl.language_models.registry import Model
-            from edsl.results.Result import Result
-            from edsl.results.Results import Results
-            from edsl.scenarios.Scenario import Scenario
-            from edsl.surveys.Survey import Survey
+            import time
+            from datetime import datetime
+            from edsl.config import CONFIG
+
+            expected_parrot_url = CONFIG.get("EXPECTED_PARROT_URL")
 
             self._output("Remote inference activated. Sending job to server...")
             if remote_cache:
@@ -474,33 +483,60 @@ class Jobs(Base):
                     "Remote caching activated. The remote cache will be used for this job."
                 )
 
-            remote_job_data = coop.remote_inference_create(
+            remote_job_creation_data = coop.remote_inference_create(
                 self,
                 description=remote_inference_description,
                 status="queued",
+                iterations=n,
             )
-            self._output("Job sent!")
-            # Create mock results object to store job data
-            results = Results(
-                survey=Survey(),
-                data=[
-                    Result(
-                        agent=Agent.example(),
-                        scenario=Scenario.example(),
-                        model=Model(),
-                        iteration=1,
-                        answer={"info": "Remote job details"},
+            time_queued = datetime.now().strftime("%m/%d/%Y %I:%M:%S %p")
+            job_uuid = remote_job_creation_data.get("uuid")
+            print(f"Remote inference started (Job uuid={job_uuid}).")
+            # print(f"Job queued at {time_queued}.")
+            job_in_queue = True
+            while job_in_queue:
+                remote_job_data = coop.remote_inference_get(job_uuid)
+                status = remote_job_data.get("status")
+                if status == "cancelled":
+                    print("\r" + " " * 80 + "\r", end="")
+                    print("Job cancelled by the user.")
+                    print(
+                        f"See {expected_parrot_url}/home/remote-inference for more details."
                     )
-                ],
-            )
-            results.add_columns_from_dict([remote_job_data])
-            if self.verbose:
-                results.select(["info", "uuid", "status", "version"]).print(
-                    format="rich"
-                )
-            return results
+                    return None
+                elif status == "failed":
+                    print("\r" + " " * 80 + "\r", end="")
+                    print("Job failed.")
+                    print(
+                        f"See {expected_parrot_url}/home/remote-inference for more details."
+                    )
+                    return None
+                elif status == "completed":
+                    results_uuid = remote_job_data.get("results_uuid")
+                    results = coop.get(results_uuid, expected_object_type="results")
+                    print("\r" + " " * 80 + "\r", end="")
+                    print(
+                        f"Job completed and Results stored on Coop (Results uuid={results_uuid})."
+                    )
+                    return results
+                else:
+                    duration = 10 if len(self) < 10 else 60
+                    time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
+                    frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
+                    start_time = time.time()
+                    i = 0
+                    while time.time() - start_time < duration:
+                        print(
+                            f"\r{frames[i % len(frames)]} Job status: {status} - last update: {time_checked}",
+                            end="",
+                            flush=True,
+                        )
+                        time.sleep(0.1)
+                        i += 1
         else:
             if check_api_keys:
+                from edsl import Model
+
                 for model in self.models + [Model()]:
                     if not model.has_valid_api_key():
                         raise MissingAPIKeyError(
@@ -606,9 +642,9 @@ class Jobs(Base):
         results = JobsRunnerAsyncio(self).run(*args, **kwargs)
         return results
 
-    async def run_async(self, cache=None, **kwargs):
+    async def run_async(self, cache=None, n=1, **kwargs):
         """Run the job asynchronously."""
-        results = await JobsRunnerAsyncio(self).run_async(cache=cache, **kwargs)
+        results = await JobsRunnerAsyncio(self).run_async(cache=cache, n=n, **kwargs)
         return results
 
     def all_question_parameters(self):
@@ -688,7 +724,10 @@ class Jobs(Base):
     #######################
     @classmethod
     def example(
-        cls, throw_exception_probability: int = 0, randomize: bool = False
+        cls,
+        throw_exception_probability: int = 0,
+        randomize: bool = False,
+        test_model=False,
     ) -> Jobs:
         """Return an example Jobs instance.
 
@@ -706,6 +745,11 @@ class Jobs(Base):
 
         addition = "" if not randomize else str(uuid4())
 
+        if test_model:
+            from edsl.language_models import LanguageModel
+
+            m = LanguageModel.example(test_model=True)
+
         # (status, question, period)
         agent_answers = {
             ("Joyful", "how_feeling", "morning"): "OK",
@@ -753,7 +797,10 @@ class Jobs(Base):
                 Scenario({"period": "afternoon"}),
             ]
         )
-        job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
+        if test_model:
+            job = base_survey.by(m).by(scenario_list).by(joy_agent, sad_agent)
+        else:
+            job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
 
         return job
 

edsl/jobs/buckets/TokenBucket.py

@@ -30,7 +30,7 @@ class TokenBucket:
         if self.turbo_mode:
             pass
         else:
-            #pass
+            # pass
             self.turbo_mode = True
             self.capacity = float("inf")
             self.refill_rate = float("inf")
@@ -74,7 +74,7 @@ class TokenBucket:
 
     def refill(self) -> None:
         """Refill the bucket with new tokens based on elapsed time.
-        
+
 
 
         >>> bucket = TokenBucket(bucket_name="test", bucket_type="test", capacity=10, refill_rate=1)
@@ -82,7 +82,7 @@ class TokenBucket:
         >>> bucket.refill()
         >>> bucket.tokens > 0
         True
-        
+
         """
         now = time.monotonic()
         elapsed = now - self.last_refill
@@ -100,7 +100,9 @@ class TokenBucket:
         available_tokens = min(self.capacity, self.tokens + refill_amount)
         return max(0, requested_tokens - available_tokens) / self.refill_rate
 
-    async def get_tokens(self, amount: Union[int, float] = 1) -> None:
+    async def get_tokens(
+        self, amount: Union[int, float] = 1, cheat_bucket_capacity=True
+    ) -> None:
         """Wait for the specified number of tokens to become available.
 
 
@@ -116,14 +118,20 @@ class TokenBucket:
         True
 
         >>> bucket = TokenBucket(bucket_name="test", bucket_type="test", capacity=10, refill_rate=1)
-        >>> asyncio.run(bucket.get_tokens(11))
+        >>> asyncio.run(bucket.get_tokens(11, cheat_bucket_capacity=False))
         Traceback (most recent call last):
         ...
         ValueError: Requested amount exceeds bucket capacity. Bucket capacity: 10, requested amount: 11. As the bucket never overflows, the requested amount will never be available.
+        >>> asyncio.run(bucket.get_tokens(11, cheat_bucket_capacity=True))
         """
         if amount > self.capacity:
-            msg = f"Requested amount exceeds bucket capacity. Bucket capacity: {self.capacity}, requested amount: {amount}. As the bucket never overflows, the requested amount will never be available."
-            raise ValueError(msg)
+            if not cheat_bucket_capacity:
+                msg = f"Requested amount exceeds bucket capacity. Bucket capacity: {self.capacity}, requested amount: {amount}. As the bucket never overflows, the requested amount will never be available."
+                raise ValueError(msg)
+            else:
+                self.tokens = 0  # clear the bucket but let it go through
+                return
+
         while self.tokens < amount:
             self.refill()
             await asyncio.sleep(0.01)  # Sleep briefly to prevent busy waiting

edsl/jobs/interviews/Interview.py

@@ -14,17 +14,19 @@ from edsl.jobs.tasks.TaskCreators import TaskCreators
 from edsl.jobs.interviews.InterviewStatusLog import InterviewStatusLog
 from edsl.jobs.interviews.interview_exception_tracking import (
     InterviewExceptionCollection,
-    InterviewExceptionEntry,
 )
+from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 from edsl.jobs.interviews.retry_management import retry_strategy
 from edsl.jobs.interviews.InterviewTaskBuildingMixin import InterviewTaskBuildingMixin
 from edsl.jobs.interviews.InterviewStatusMixin import InterviewStatusMixin
 
 import asyncio
 
+
 def run_async(coro):
     return asyncio.run(coro)
 
+
 class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
     """
     An 'interview' is one agent answering one survey, with one language model, for a given scenario.
@@ -41,7 +43,8 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         debug: Optional[bool] = False,
         iteration: int = 0,
         cache: Optional["Cache"] = None,
-        sidecar_model: Optional['LanguageModel'] = None,
+        sidecar_model: Optional["LanguageModel"] = None,
+        skip_retry=False,
     ):
         """Initialize the Interview instance.
 
@@ -85,6 +88,7 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         self.task_creators = TaskCreators()  # tracks the task creators
         self.exceptions = InterviewExceptionCollection()
         self._task_status_log_dict = InterviewStatusLog()
+        self.skip_retry = skip_retry
 
         # dictionary mapping question names to their index in the survey.
         self.to_index = {
@@ -92,13 +96,37 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
             for index, question_name in enumerate(self.survey.question_names)
         }
 
+    def _to_dict(self, include_exceptions=False) -> dict[str, Any]:
+        """Return a dictionary representation of the Interview instance.
+        This is just for hashing purposes.
+
+        >>> i = Interview.example()
+        >>> hash(i)
+        1646262796627658719
+        """
+        d = {
+            "agent": self.agent._to_dict(),
+            "survey": self.survey._to_dict(),
+            "scenario": self.scenario._to_dict(),
+            "model": self.model._to_dict(),
+            "iteration": self.iteration,
+            "exceptions": {},
+        }
+        if include_exceptions:
+            d["exceptions"] = self.exceptions.to_dict()
+
+    def __hash__(self) -> int:
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self._to_dict())
+
     async def async_conduct_interview(
         self,
         *,
         model_buckets: ModelBuckets = None,
         debug: bool = False,
         stop_on_exception: bool = False,
-        sidecar_model: Optional['LanguageModel'] = None,
+        sidecar_model: Optional["LanguageModel"] = None,
     ) -> tuple["Answers", List[dict[str, Any]]]:
         """
         Conduct an Interview asynchronously.
@@ -132,8 +160,7 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         <BLANKLINE>
 
         >>> i.exceptions
-        {'q0': [{'exception': "Exception('This is a test error')", 'time': ..., 'traceback': ...
-
+        {'q0': ...
         >>> i = Interview.example()
         >>> result, _ = asyncio.run(i.async_conduct_interview(stop_on_exception = True))
         Traceback (most recent call last):
@@ -146,13 +173,11 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         if model_buckets is None or hasattr(self.agent, "answer_question_directly"):
             model_buckets = ModelBuckets.infinity_bucket()
 
-        
         ## build the tasks using the InterviewTaskBuildingMixin
         ## This is the key part---it creates a task for each question,
         ## with dependencies on the questions that must be answered before this one can be answered.
         self.tasks = self._build_question_tasks(
-            debug=debug, 
-            model_buckets=model_buckets
+            debug=debug, model_buckets=model_buckets
         )
 
         ## 'Invigilators' are used to administer the survey
@@ -195,7 +220,7 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
 
     def _record_exception(self, task, exception: Exception) -> None:
         """Record an exception in the Interview instance.
-        
+
         It records the exception in the Interview instance, with the task name and the exception entry.
 
         >>> i = Interview.example()
@@ -204,13 +229,9 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         {}
         >>> i._record_exception(i.tasks[0], Exception("An exception occurred."))
         >>> i.exceptions
-        {'q0': [{'exception': "Exception('An exception occurred.')", 'time': ..., 'traceback': 'NoneType: None\\n'}]}
+        {'q0': ...
         """
-        exception_entry = InterviewExceptionEntry(
-            exception=repr(exception),
-            time=time.time(),
-            traceback=traceback.format_exc(),
-        )
+        exception_entry = InterviewExceptionEntry(exception)
         self.exceptions.add(task.get_name(), exception_entry)
 
     @property
@@ -235,14 +256,14 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         """Return a string representation of the Interview instance."""
         return f"Interview(agent = {repr(self.agent)}, survey = {repr(self.survey)}, scenario = {repr(self.scenario)}, model = {repr(self.model)})"
 
-    def duplicate(self, iteration: int, cache: 'Cache') -> Interview:
+    def duplicate(self, iteration: int, cache: "Cache") -> Interview:
         """Duplicate the interview, but with a new iteration number and cache.
-        
+
         >>> i = Interview.example()
         >>> i2 = i.duplicate(1, None)
         >>> i.iteration + 1 == i2.iteration
         True
-        
+
         """
         return Interview(
             agent=self.agent,
@@ -251,6 +272,7 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
             model=self.model,
             iteration=iteration,
             cache=cache,
+            skip_retry=self.skip_retry,
         )
 
     @classmethod
@@ -270,7 +292,7 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         scenario = Scenario.example()
         model = LanguageModel.example()
         if throw_exception:
-            model = LanguageModel.example(test_model = True, throw_exception=True)
+            model = LanguageModel.example(test_model=True, throw_exception=True)
             agent = Agent.example()
             return Interview(agent=agent, survey=survey, scenario=scenario, model=model)
         return Interview(agent=agent, survey=survey, scenario=scenario, model=model)

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -0,0 +1,101 @@
+import traceback
+import datetime
+import time
+from collections import UserDict
+
+# traceback=traceback.format_exc(),
+# traceback = frame_summary_to_dict(traceback.extract_tb(e.__traceback__))
+# traceback = [frame_summary_to_dict(f) for f in traceback.extract_tb(e.__traceback__)]
+
+
+class InterviewExceptionEntry:
+    """Class to record an exception that occurred during the interview.
+
+    >>> entry = InterviewExceptionEntry.example()
+    >>> entry.to_dict()['exception']
+    "ValueError('An error occurred.')"
+    """
+
+    def __init__(self, exception: Exception, traceback_format="html"):
+        self.time = datetime.datetime.now().isoformat()
+        self.exception = exception
+        self.traceback_format = traceback_format
+
+    def __getitem__(self, key):
+        # Support dict-like access obj['a']
+        return str(getattr(self, key))
+
+    @classmethod
+    def example(cls):
+        try:
+            raise ValueError("An error occurred.")
+        except Exception as e:
+            entry = InterviewExceptionEntry(e)
+        return entry
+
+    @property
+    def traceback(self):
+        """Return the exception as HTML."""
+        if self.traceback_format == "html":
+            return self.html_traceback
+        else:
+            return self.text_traceback
+
+    @property
+    def text_traceback(self):
+        """
+        >>> entry = InterviewExceptionEntry.example()
+        >>> entry.text_traceback
+        'Traceback (most recent call last):...'
+        """
+        e = self.exception
+        tb_str = "".join(traceback.format_exception(type(e), e, e.__traceback__))
+        return tb_str
+
+    @property
+    def html_traceback(self):
+        from rich.console import Console
+        from rich.table import Table
+        from rich.traceback import Traceback
+
+        from io import StringIO
+
+        html_output = StringIO()
+
+        console = Console(file=html_output, record=True)
+
+        tb = Traceback.from_exception(
+            type(self.exception),
+            self.exception,
+            self.exception.__traceback__,
+            show_locals=True,
+        )
+        console.print(tb)
+        return html_output.getvalue()
+
+    def to_dict(self) -> dict:
+        """Return the exception as a dictionary.
+
+        >>> entry = InterviewExceptionEntry.example()
+        >>> entry.to_dict()['exception']
+        "ValueError('An error occurred.')"
+
+        """
+        return {
+            "exception": repr(self.exception),
+            "time": self.time,
+            "traceback": self.traceback,
+        }
+
+    def push(self):
+        from edsl import Coop
+
+        coop = Coop()
+        results = coop.error_create(self.to_dict())
+        return results
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)