edsl 0.1.36.dev3__tar.gz → 0.1.36.dev4__tar.gz

{edsl-0.1.36.dev3 → edsl-0.1.36.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edsl
-Version: 0.1.36.dev3
+Version: 0.1.36.dev4
 Summary: Create and analyze LLM-based surveys
 Home-page: https://www.expectedparrot.com/
 License: MIT

edsl-0.1.36.dev4/edsl/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.36.dev4"

{edsl-0.1.36.dev3 → edsl-0.1.36.dev4}/edsl/coop/coop.py

@@ -6,6 +6,7 @@ from typing import Any, Optional, Union, Literal
 from uuid import UUID
 import edsl
 from edsl import CONFIG, CacheEntry, Jobs, Survey
+from edsl.exceptions.coop import CoopNoUUIDError, CoopServerResponseError
 from edsl.coop.utils import (
     EDSLObject,
     ObjectRegistry,
@@ -99,7 +100,7 @@ class Coop:
             if "Authorization" in message:
                 print(message)
                 message = "Please provide an Expected Parrot API key."
-            raise Exception(message)
+            raise CoopServerResponseError(message)
 
     def _json_handle_none(self, value: Any) -> Any:
         """
@@ -116,7 +117,7 @@ class Coop:
         Resolve the uuid from a uuid or a url.
         """
         if not url and not uuid:
-            raise Exception("No uuid or url provided for the object.")
+            raise CoopNoUUIDError("No uuid or url provided for the object.")
         if not uuid and url:
             uuid = url.split("/")[-1]
         return uuid

edsl-0.1.36.dev4/edsl/data/RemoteCacheSync.py

@@ -0,0 +1,84 @@
+class RemoteCacheSync:
+    def __init__(self, coop, cache, output_func, remote_cache=True, remote_cache_description=""):
+        self.coop = coop
+        self.cache = cache
+        self._output = output_func
+        self.remote_cache = remote_cache
+        self.old_entry_keys = []
+        self.new_cache_entries = []
+        self.remote_cache_description = remote_cache_description
+
+    def __enter__(self):
+        if self.remote_cache:
+            self._sync_from_remote()
+            self.old_entry_keys = list(self.cache.keys())
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.remote_cache:
+            self._sync_to_remote()
+        return False  # Propagate exceptions
+
+    def _sync_from_remote(self):
+        cache_difference = self.coop.remote_cache_get_diff(self.cache.keys())
+        client_missing_cacheentries = cache_difference.get("client_missing_cacheentries", [])
+        missing_entry_count = len(client_missing_cacheentries)
+        
+        if missing_entry_count > 0:
+            self._output(
+                f"Updating local cache with {missing_entry_count:,} new "
+                f"{'entry' if missing_entry_count == 1 else 'entries'} from remote..."
+            )
+            self.cache.add_from_dict({entry.key: entry for entry in client_missing_cacheentries})
+            self._output("Local cache updated!")
+        else:
+            self._output("No new entries to add to local cache.")
+
+    def _sync_to_remote(self):
+        cache_difference = self.coop.remote_cache_get_diff(self.cache.keys())
+        server_missing_cacheentry_keys = cache_difference.get("server_missing_cacheentry_keys", [])
+        server_missing_cacheentries = [
+            entry
+            for key in server_missing_cacheentry_keys
+            if (entry := self.cache.data.get(key)) is not None
+        ]
+        
+        new_cache_entries = [
+            entry for entry in self.cache.values() if entry.key not in self.old_entry_keys
+        ]
+        server_missing_cacheentries.extend(new_cache_entries)
+        new_entry_count = len(server_missing_cacheentries)
+        
+        if new_entry_count > 0:
+            self._output(
+                f"Updating remote cache with {new_entry_count:,} new "
+                f"{'entry' if new_entry_count == 1 else 'entries'}..."
+            )
+            self.coop.remote_cache_create_many(
+                server_missing_cacheentries,
+                visibility="private",
+                description=self.remote_cache_description,
+            )
+            self._output("Remote cache updated!")
+        else:
+            self._output("No new entries to add to remote cache.")
+        
+        self._output(f"There are {len(self.cache.keys()):,} entries in the local cache.")
+
+# # Usage example
+# def run_job(self, n, progress_bar, cache, stop_on_exception, sidecar_model, print_exceptions, raise_validation_errors, use_remote_cache=True):
+#     with RemoteCacheSync(self.coop, cache, self._output, remote_cache=use_remote_cache):
+#         self._output("Running job...")
+#         results = self._run_local(
+#             n=n,
+#             progress_bar=progress_bar,
+#             cache=cache,
+#             stop_on_exception=stop_on_exception,
+#             sidecar_model=sidecar_model,
+#             print_exceptions=print_exceptions,
+#             raise_validation_errors=raise_validation_errors,
+#         )
+#         self._output("Job completed!")
+    
+#     results.cache = cache.new_entries_cache()
+#     return results

edsl-0.1.36.dev4/edsl/exceptions/coop.py

@@ -0,0 +1,10 @@
+class CoopErrors(Exception):
+    pass
+
+
+class CoopNoUUIDError(CoopErrors):
+    pass
+
+
+class CoopServerResponseError(CoopErrors):
+    pass

{edsl-0.1.36.dev3 → edsl-0.1.36.dev4}/edsl/jobs/Jobs.py

@@ -1,8 +1,10 @@
 # """The Jobs class is a collection of agents, scenarios and models and one survey."""
 from __future__ import annotations
 import warnings
+import requests
 from itertools import product
 from typing import Optional, Union, Sequence, Generator
+
 from edsl.Base import Base
 from edsl.exceptions import MissingAPIKeyError
 from edsl.jobs.buckets.BucketCollection import BucketCollection
@@ -10,6 +12,9 @@ from edsl.jobs.interviews.Interview import Interview
 from edsl.jobs.runners.JobsRunnerAsyncio import JobsRunnerAsyncio
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
+from edsl.data.RemoteCacheSync import RemoteCacheSync
+from edsl.exceptions.coop import CoopServerResponseError
+
 
 class Jobs(Base):
     """
@@ -203,10 +208,6 @@ class Jobs(Base):
             ]
         )
         return d
-        # if table:
-        #     d.to_scenario_list().print(format="rich")
-        # else:
-        #     return d
 
     def show_prompts(self) -> None:
         """Print the prompts."""
@@ -615,7 +616,7 @@ class Jobs(Base):
 
     def _output(self, message) -> None:
         """Check if a Job is verbose. If so, print the message."""
-        if self.verbose:
+        if hasattr(self, "verbose") and self.verbose:
             print(message)
 
     def _check_parameters(self, strict=False, warn=False) -> None:
@@ -692,6 +693,122 @@ class Jobs(Base):
             return False
         return self._raise_validation_errors
 
+    def create_remote_inference_job(
+        self, iterations: int = 1, remote_inference_description: Optional[str] = None
+    ):
+        """ """
+        from edsl.coop.coop import Coop
+
+        coop = Coop()
+        self._output("Remote inference activated. Sending job to server...")
+        remote_job_creation_data = coop.remote_inference_create(
+            self,
+            description=remote_inference_description,
+            status="queued",
+            iterations=iterations,
+        )
+        return remote_job_creation_data
+
+    @staticmethod
+    def check_status(job_uuid):
+        from edsl.coop.coop import Coop
+
+        coop = Coop()
+        return coop.remote_inference_get(job_uuid)
+
+    def poll_remote_inference_job(
+        self, remote_job_creation_data: dict
+    ) -> Union[Results, None]:
+        from edsl.coop.coop import Coop
+        import time
+        from datetime import datetime
+        from edsl.config import CONFIG
+
+        expected_parrot_url = CONFIG.get("EXPECTED_PARROT_URL")
+
+        job_uuid = remote_job_creation_data.get("uuid")
+
+        coop = Coop()
+        job_in_queue = True
+        while job_in_queue:
+            remote_job_data = coop.remote_inference_get(job_uuid)
+            status = remote_job_data.get("status")
+            if status == "cancelled":
+                print("\r" + " " * 80 + "\r", end="")
+                print("Job cancelled by the user.")
+                print(
+                    f"See {expected_parrot_url}/home/remote-inference for more details."
+                )
+                return None
+            elif status == "failed":
+                print("\r" + " " * 80 + "\r", end="")
+                print("Job failed.")
+                print(
+                    f"See {expected_parrot_url}/home/remote-inference for more details."
+                )
+                return None
+            elif status == "completed":
+                results_uuid = remote_job_data.get("results_uuid")
+                results = coop.get(results_uuid, expected_object_type="results")
+                print("\r" + " " * 80 + "\r", end="")
+                print(
+                    f"Job completed and Results stored on Coop (Results uuid={results_uuid})."
+                )
+                return results
+            else:
+                duration = 5
+                time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
+                frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
+                start_time = time.time()
+                i = 0
+                while time.time() - start_time < duration:
+                    print(
+                        f"\r{frames[i % len(frames)]} Job status: {status} - last update: {time_checked}",
+                        end="",
+                        flush=True,
+                    )
+                    time.sleep(0.1)
+                    i += 1
+
+    def use_remote_inference(self, disable_remote_inference: bool):
+        if disable_remote_inference:
+            return False
+        if not disable_remote_inference:
+            try:
+                from edsl import Coop
+
+                user_edsl_settings = Coop().edsl_settings
+                return user_edsl_settings.get("remote_inference", False)
+            except requests.ConnectionError:
+                pass
+            except CoopServerResponseError as e:
+                pass
+
+        return False
+
+    def use_remote_cache(self):
+        try:
+            from edsl import Coop
+
+            user_edsl_settings = Coop().edsl_settings
+            return user_edsl_settings.get("remote_caching", False)
+        except requests.ConnectionError:
+            pass
+        except CoopServerResponseError as e:
+            pass
+
+        return False
+
+    def check_api_keys(self):
+        from edsl import Model
+
+        for model in self.models + [Model()]:
+            if not model.has_valid_api_key():
+                raise MissingAPIKeyError(
+                    model_name=str(model.model),
+                    inference_service=model._inference_service_,
+                )
+
     def run(
         self,
         n: int = 1,
@@ -729,91 +846,17 @@ class Jobs(Base):
 
         self.verbose = verbose
 
-        remote_cache = False
-        remote_inference = False
-
-        if not disable_remote_inference:
-            try:
-                coop = Coop()
-                user_edsl_settings = Coop().edsl_settings
-                remote_cache = user_edsl_settings.get("remote_caching", False)
-                remote_inference = user_edsl_settings.get("remote_inference", False)
-            except Exception:
-                pass
-
-        if remote_inference:
-            import time
-            from datetime import datetime
-            from edsl.config import CONFIG
-
-            expected_parrot_url = CONFIG.get("EXPECTED_PARROT_URL")
-
-            self._output("Remote inference activated. Sending job to server...")
-            if remote_cache:
-                self._output(
-                    "Remote caching activated. The remote cache will be used for this job."
-                )
-
-            remote_job_creation_data = coop.remote_inference_create(
-                self,
-                description=remote_inference_description,
-                status="queued",
-                iterations=n,
+        if remote_inference := self.use_remote_inference(disable_remote_inference):
+            remote_job_creation_data = self.create_remote_inference_job(
+                iterations=n, remote_inference_description=remote_inference_description
             )
-            time_queued = datetime.now().strftime("%m/%d/%Y %I:%M:%S %p")
-            job_uuid = remote_job_creation_data.get("uuid")
-            print(f"Remote inference started (Job uuid={job_uuid}).")
-            # print(f"Job queued at {time_queued}.")
-            job_in_queue = True
-            while job_in_queue:
-                remote_job_data = coop.remote_inference_get(job_uuid)
-                status = remote_job_data.get("status")
-                if status == "cancelled":
-                    print("\r" + " " * 80 + "\r", end="")
-                    print("Job cancelled by the user.")
-                    print(
-                        f"See {expected_parrot_url}/home/remote-inference for more details."
-                    )
-                    return None
-                elif status == "failed":
-                    print("\r" + " " * 80 + "\r", end="")
-                    print("Job failed.")
-                    print(
-                        f"See {expected_parrot_url}/home/remote-inference for more details."
-                    )
-                    return None
-                elif status == "completed":
-                    results_uuid = remote_job_data.get("results_uuid")
-                    results = coop.get(results_uuid, expected_object_type="results")
-                    print("\r" + " " * 80 + "\r", end="")
-                    print(
-                        f"Job completed and Results stored on Coop (Results uuid={results_uuid})."
-                    )
-                    return results
-                else:
-                    duration = 5
-                    time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
-                    frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
-                    start_time = time.time()
-                    i = 0
-                    while time.time() - start_time < duration:
-                        print(
-                            f"\r{frames[i % len(frames)]} Job status: {status} - last update: {time_checked}",
-                            end="",
-                            flush=True,
-                        )
-                        time.sleep(0.1)
-                        i += 1
-        else:
-            if check_api_keys:
-                from edsl import Model
+            results = self.poll_remote_inference_job(remote_job_creation_data)
+            if results is None:
+                self._output("Job failed.")
+            return results
 
-                for model in self.models + [Model()]:
-                    if not model.has_valid_api_key():
-                        raise MissingAPIKeyError(
-                            model_name=str(model.model),
-                            inference_service=model._inference_service_,
-                        )
+        if check_api_keys:
+            self.check_api_keys()
 
         # handle cache
         if cache is None or cache is True:
@@ -825,7 +868,14 @@ class Jobs(Base):
 
             cache = Cache()
 
-        if not remote_cache:
+        remote_cache = self.use_remote_cache()
+        with RemoteCacheSync(
+            coop=Coop(),
+            cache=cache,
+            output_func=self._output,
+            remote_cache=remote_cache,
+            remote_cache_description=remote_cache_description,
+        ) as r:
             results = self._run_local(
                 n=n,
                 progress_bar=progress_bar,
@@ -836,75 +886,7 @@ class Jobs(Base):
                 raise_validation_errors=raise_validation_errors,
             )
 
-            results.cache = cache.new_entries_cache()
-
-            self._output(f"There are {len(cache.keys()):,} entries in the local cache.")
-        else:
-            cache_difference = coop.remote_cache_get_diff(cache.keys())
-
-            client_missing_cacheentries = cache_difference.get(
-                "client_missing_cacheentries", []
-            )
-
-            missing_entry_count = len(client_missing_cacheentries)
-            if missing_entry_count > 0:
-                self._output(
-                    f"Updating local cache with {missing_entry_count:,} new "
-                    f"{'entry' if missing_entry_count == 1 else 'entries'} from remote..."
-                )
-                cache.add_from_dict(
-                    {entry.key: entry for entry in client_missing_cacheentries}
-                )
-                self._output("Local cache updated!")
-            else:
-                self._output("No new entries to add to local cache.")
-
-            server_missing_cacheentry_keys = cache_difference.get(
-                "server_missing_cacheentry_keys", []
-            )
-            server_missing_cacheentries = [
-                entry
-                for key in server_missing_cacheentry_keys
-                if (entry := cache.data.get(key)) is not None
-            ]
-            old_entry_keys = [key for key in cache.keys()]
-
-            self._output("Running job...")
-            results = self._run_local(
-                n=n,
-                progress_bar=progress_bar,
-                cache=cache,
-                stop_on_exception=stop_on_exception,
-                sidecar_model=sidecar_model,
-                print_exceptions=print_exceptions,
-                raise_validation_errors=raise_validation_errors,
-            )
-            self._output("Job completed!")
-
-            new_cache_entries = list(
-                [entry for entry in cache.values() if entry.key not in old_entry_keys]
-            )
-            server_missing_cacheentries.extend(new_cache_entries)
-
-            new_entry_count = len(server_missing_cacheentries)
-            if new_entry_count > 0:
-                self._output(
-                    f"Updating remote cache with {new_entry_count:,} new "
-                    f"{'entry' if new_entry_count == 1 else 'entries'}..."
-                )
-                coop.remote_cache_create_many(
-                    server_missing_cacheentries,
-                    visibility="private",
-                    description=remote_cache_description,
-                )
-                self._output("Remote cache updated!")
-            else:
-                self._output("No new entries to add to remote cache.")
-
-            results.cache = cache.new_entries_cache()
-
-            self._output(f"There are {len(cache.keys()):,} entries in the local cache.")
-
+        results.cache = cache.new_entries_cache()
         return results
 
     def _run_local(self, *args, **kwargs):

{edsl-0.1.36.dev3 → edsl-0.1.36.dev4}/edsl/jobs/interviews/Interview.py

@@ -159,13 +159,13 @@ class Interview:
         return self.task_creators.interview_status
 
     # region: Serialization
-    def _to_dict(self, include_exceptions=False) -> dict[str, Any]:
+    def _to_dict(self, include_exceptions=True) -> dict[str, Any]:
         """Return a dictionary representation of the Interview instance.
         This is just for hashing purposes.
 
         >>> i = Interview.example()
         >>> hash(i)
-        1646262796627658719
+        1217840301076717434
         """
         d = {
             "agent": self.agent._to_dict(),
@@ -177,11 +177,29 @@ class Interview:
         }
         if include_exceptions:
             d["exceptions"] = self.exceptions.to_dict()
+        return d
+    
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "Interview":
+        """Return an Interview instance from a dictionary."""
+        agent = Agent.from_dict(d["agent"])
+        survey = Survey.from_dict(d["survey"])
+        scenario = Scenario.from_dict(d["scenario"])
+        model = LanguageModel.from_dict(d["model"])
+        iteration = d["iteration"]
+        return cls(agent=agent, survey=survey, scenario=scenario, model=model, iteration=iteration)
 
     def __hash__(self) -> int:
         from edsl.utilities.utilities import dict_hash
 
-        return dict_hash(self._to_dict())
+        return dict_hash(self._to_dict(include_exceptions=False))
+    
+    def __eq__(self, other: "Interview") -> bool:
+        """
+        >>> from edsl.jobs.interviews.Interview import Interview; i = Interview.example(); d = i._to_dict(); i2 = Interview.from_dict(d); i == i2
+        True
+        """
+        return hash(self) == hash(other)
 
     # endregion
 

{edsl-0.1.36.dev3 → edsl-0.1.36.dev4}/edsl/jobs/interviews/InterviewExceptionCollection.py

@@ -33,6 +33,15 @@ class InterviewExceptionCollection(UserDict):
         """Return the collection of exceptions as a dictionary."""
         newdata = {k: [e.to_dict() for e in v] for k, v in self.data.items()}
         return newdata
+    
+    @classmethod
+    def from_dict(cls, data: dict) -> "InterviewExceptionCollection":
+        """Create an InterviewExceptionCollection from a dictionary."""
+        collection = cls()
+        for question_name, entries in data.items():
+            for entry in entries:
+                collection.add(question_name, InterviewExceptionEntry.from_dict(entry))
+        return collection
 
     def _repr_html_(self) -> str:
         from edsl.utilities.utilities import data_to_html

{edsl-0.1.36.dev3 → edsl-0.1.36.dev4}/edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -9,7 +9,6 @@ class InterviewExceptionEntry:
         self,
         *,
         exception: Exception,
-        # failed_question: FailedQuestion,
         invigilator: "Invigilator",
         traceback_format="text",
         answers=None,
@@ -133,22 +132,41 @@ class InterviewExceptionEntry:
         )
         console.print(tb)
         return html_output.getvalue()
+    
+    @staticmethod
+    def serialize_exception(exception: Exception) -> dict:
+        return {
+            "type": type(exception).__name__,
+            "message": str(exception),
+            "traceback": "".join(traceback.format_exception(type(exception), exception, exception.__traceback__)),
+        }
+    
+    @staticmethod
+    def deserialize_exception(data: dict) -> Exception:
+        exception_class = globals()[data["type"]]
+        return exception_class(data["message"])
 
     def to_dict(self) -> dict:
         """Return the exception as a dictionary.
 
         >>> entry = InterviewExceptionEntry.example()
-        >>> entry.to_dict()['exception']
-        ValueError()
-
+        >>> _ = entry.to_dict()
         """
         return {
-            "exception": self.exception,
+            "exception": self.serialize_exception(self.exception),
             "time": self.time,
             "traceback": self.traceback,
-            # "failed_question": self.failed_question.to_dict(),
             "invigilator": self.invigilator.to_dict(),
         }
+    
+    @classmethod
+    def from_dict(cls, data: dict) -> "InterviewExceptionEntry":
+        """Create an InterviewExceptionEntry from a dictionary."""
+        from edsl.agents.Invigilator import InvigilatorAI
+
+        exception = cls.deserialize_exception(data["exception"])
+        invigilator = InvigilatorAI.from_dict(data["invigilator"])
+        return cls(exception=exception, invigilator=invigilator)
 
     def push(self):
         from edsl import Coop

{edsl-0.1.36.dev3 → edsl-0.1.36.dev4}/edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -1,18 +1,12 @@
 from __future__ import annotations
 import time
-import math
 import asyncio
-import functools
 import threading
 from typing import Coroutine, List, AsyncGenerator, Optional, Union, Generator
 from contextlib import contextmanager
 from collections import UserList
 
-from rich.live import Live
-from rich.console import Console
-
 from edsl.results.Results import Results
-from edsl import shared_globals
 from edsl.jobs.interviews.Interview import Interview
 from edsl.jobs.runners.JobsRunnerStatus import JobsRunnerStatus
 
@@ -25,7 +19,6 @@ from edsl.results.Results import Results
 from edsl.language_models.LanguageModel import LanguageModel
 from edsl.data.Cache import Cache
 
-
 class StatusTracker(UserList):
     def __init__(self, total_tasks: int):
         self.total_tasks = total_tasks
@@ -48,8 +41,6 @@ class JobsRunnerAsyncio:
         self.bucket_collection: "BucketCollection" = jobs.bucket_collection
         self.total_interviews: List["Interview"] = []
 
-        # self.jobs_runner_status = JobsRunnerStatus(self, n=1)
-
     async def run_async_generator(
         self,
         cache: Cache,
@@ -228,17 +219,16 @@ class JobsRunnerAsyncio:
         }
         interview_hashes = list(interview_lookup.keys())
 
+        task_history = TaskHistory(self.total_interviews, include_traceback=False)
+
         results = Results(
             survey=self.jobs.survey,
             data=sorted(
                 raw_results, key=lambda x: interview_hashes.index(x.interview_hash)
             ),
+            task_history=task_history,
+            cache=cache,
         )
-        results.cache = cache
-        results.task_history = TaskHistory(
-            self.total_interviews, include_traceback=False
-        )
-        results.has_unfixed_exceptions = results.task_history.has_unfixed_exceptions
         results.bucket_collection = self.bucket_collection
 
         if results.has_unfixed_exceptions and print_exceptions:
@@ -266,6 +256,7 @@ class JobsRunnerAsyncio:
             except Exception as e:
                 print(e)
                 remote_logging = False
+
             if remote_logging:
                 filestore = HTMLFileStore(filepath)
                 coop_details = filestore.push(description="Error report")