edsl 0.1.36.dev1__py3-none-any.whl → 0.1.36.dev5__py3-none-any.whl

edsl/jobs/Jobs.py

@@ -1,8 +1,10 @@
 # """The Jobs class is a collection of agents, scenarios and models and one survey."""
 from __future__ import annotations
 import warnings
+import requests
 from itertools import product
 from typing import Optional, Union, Sequence, Generator
+
 from edsl.Base import Base
 from edsl.exceptions import MissingAPIKeyError
 from edsl.jobs.buckets.BucketCollection import BucketCollection
@@ -10,6 +12,9 @@ from edsl.jobs.interviews.Interview import Interview
 from edsl.jobs.runners.JobsRunnerAsyncio import JobsRunnerAsyncio
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
+from edsl.data.RemoteCacheSync import RemoteCacheSync
+from edsl.exceptions.coop import CoopServerResponseError
+
 
 class Jobs(Base):
     """
@@ -203,14 +208,15 @@ class Jobs(Base):
             ]
         )
         return d
-        # if table:
-        #     d.to_scenario_list().print(format="rich")
-        # else:
-        #     return d
 
-    def show_prompts(self) -> None:
+    def show_prompts(self, all=False) -> None:
         """Print the prompts."""
-        self.prompts().to_scenario_list().print(format="rich")
+        if all:
+            self.prompts().to_scenario_list().print(format="rich")
+        else:
+            self.prompts().select(
+                "user_prompt", "system_prompt"
+            ).to_scenario_list().print(format="rich")
 
     @staticmethod
     def estimate_prompt_cost(
@@ -219,11 +225,11 @@ class Jobs(Base):
         price_lookup: dict,
         inference_service: str,
         model: str,
-    ):
+    ) -> dict:
         """Estimates the cost of a prompt. Takes piping into account."""
 
         def get_piping_multiplier(prompt: str):
-            """Returns 2 if a prompt includes Jinja brances, and 1 otherwise."""
+            """Returns 2 if a prompt includes Jinja braces, and 1 otherwise."""
 
             if "{{" in prompt and "}}" in prompt:
                 return 2
@@ -231,9 +237,25 @@ class Jobs(Base):
 
         # Look up prices per token
         key = (inference_service, model)
-        relevant_prices = price_lookup[key]
-        output_price_per_token = 1 / float(relevant_prices["output"]["one_usd_buys"])
-        input_price_per_token = 1 / float(relevant_prices["input"]["one_usd_buys"])
+
+        try:
+            relevant_prices = price_lookup[key]
+            output_price_per_token = 1 / float(
+                relevant_prices["output"]["one_usd_buys"]
+            )
+            input_price_per_token = 1 / float(relevant_prices["input"]["one_usd_buys"])
+        except KeyError:
+            # A KeyError is likely to occur if we cannot retrieve prices (the price_lookup dict is empty)
+            # Use a sensible default
+
+            import warnings
+
+            warnings.warn(
+                "Price data could not be retrieved. Using default estimates for input and output token prices. Input: $0.15 / 1M tokens; Output: $0.60 / 1M tokens"
+            )
+
+            output_price_per_token = 0.00000015  # $0.15 / 1M tokens
+            input_price_per_token = 0.00000060  # $0.60 / 1M tokens
 
         # Compute the number of characters (double if the question involves piping)
         user_prompt_chars = len(str(user_prompt)) * get_piping_multiplier(
@@ -258,7 +280,7 @@ class Jobs(Base):
             "cost": cost,
         }
 
-    def estimate_job_cost_from_external_prices(self, price_lookup: dict):
+    def estimate_job_cost_from_external_prices(self, price_lookup: dict) -> dict:
         """
         Estimates the cost of a job according to the following assumptions:
 
@@ -341,7 +363,7 @@ class Jobs(Base):
 
         return output
 
-    def estimate_job_cost(self):
+    def estimate_job_cost(self) -> dict:
         """
         Estimates the cost of a job according to the following assumptions:
 
@@ -357,6 +379,25 @@ class Jobs(Base):
 
         return self.estimate_job_cost_from_external_prices(price_lookup=price_lookup)
 
+    @staticmethod
+    def compute_job_cost(job_results: "Results") -> float:
+        """
+        Computes the cost of a completed job in USD.
+        """
+        total_cost = 0
+        for result in job_results:
+            for key in result.raw_model_response:
+                if key.endswith("_cost"):
+                    result_cost = result.raw_model_response[key]
+
+                    question_name = key.removesuffix("_cost")
+                    cache_used = result.cache_used_dict[question_name]
+
+                    if isinstance(result_cost, (int, float)) and not cache_used:
+                        total_cost += result_cost
+
+        return total_cost
+
     @staticmethod
     def _get_container_class(object):
         from edsl.agents.AgentList import AgentList
@@ -580,7 +621,7 @@ class Jobs(Base):
 
     def _output(self, message) -> None:
         """Check if a Job is verbose. If so, print the message."""
-        if self.verbose:
+        if hasattr(self, "verbose") and self.verbose:
             print(message)
 
     def _check_parameters(self, strict=False, warn=False) -> None:
@@ -657,6 +698,123 @@ class Jobs(Base):
             return False
         return self._raise_validation_errors
 
+    def create_remote_inference_job(
+        self, iterations: int = 1, remote_inference_description: Optional[str] = None
+    ):
+        """ """
+        from edsl.coop.coop import Coop
+
+        coop = Coop()
+        self._output("Remote inference activated. Sending job to server...")
+        remote_job_creation_data = coop.remote_inference_create(
+            self,
+            description=remote_inference_description,
+            status="queued",
+            iterations=iterations,
+        )
+        job_uuid = remote_job_creation_data.get("uuid")
+        print(f"Job sent to server. (Job uuid={job_uuid}).")
+        return remote_job_creation_data
+
+    @staticmethod
+    def check_status(job_uuid):
+        from edsl.coop.coop import Coop
+
+        coop = Coop()
+        return coop.remote_inference_get(job_uuid)
+
+    def poll_remote_inference_job(
+        self, remote_job_creation_data: dict
+    ) -> Union[Results, None]:
+        from edsl.coop.coop import Coop
+        import time
+        from datetime import datetime
+        from edsl.config import CONFIG
+
+        expected_parrot_url = CONFIG.get("EXPECTED_PARROT_URL")
+
+        job_uuid = remote_job_creation_data.get("uuid")
+
+        coop = Coop()
+        job_in_queue = True
+        while job_in_queue:
+            remote_job_data = coop.remote_inference_get(job_uuid)
+            status = remote_job_data.get("status")
+            if status == "cancelled":
+                print("\r" + " " * 80 + "\r", end="")
+                print("Job cancelled by the user.")
+                print(
+                    f"See {expected_parrot_url}/home/remote-inference for more details."
+                )
+                return None
+            elif status == "failed":
+                print("\r" + " " * 80 + "\r", end="")
+                print("Job failed.")
+                print(
+                    f"See {expected_parrot_url}/home/remote-inference for more details."
+                )
+                return None
+            elif status == "completed":
+                results_uuid = remote_job_data.get("results_uuid")
+                results = coop.get(results_uuid, expected_object_type="results")
+                print("\r" + " " * 80 + "\r", end="")
+                url = f"{expected_parrot_url}/content/{results_uuid}"
+                print(f"Job completed and Results stored on Coop: {url}.")
+                return results
+            else:
+                duration = 5
+                time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
+                frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
+                start_time = time.time()
+                i = 0
+                while time.time() - start_time < duration:
+                    print(
+                        f"\r{frames[i % len(frames)]} Job status: {status} - last update: {time_checked}",
+                        end="",
+                        flush=True,
+                    )
+                    time.sleep(0.1)
+                    i += 1
+
+    def use_remote_inference(self, disable_remote_inference: bool):
+        if disable_remote_inference:
+            return False
+        if not disable_remote_inference:
+            try:
+                from edsl import Coop
+
+                user_edsl_settings = Coop().edsl_settings
+                return user_edsl_settings.get("remote_inference", False)
+            except requests.ConnectionError:
+                pass
+            except CoopServerResponseError as e:
+                pass
+
+        return False
+
+    def use_remote_cache(self):
+        try:
+            from edsl import Coop
+
+            user_edsl_settings = Coop().edsl_settings
+            return user_edsl_settings.get("remote_caching", False)
+        except requests.ConnectionError:
+            pass
+        except CoopServerResponseError as e:
+            pass
+
+        return False
+
+    def check_api_keys(self):
+        from edsl import Model
+
+        for model in self.models + [Model()]:
+            if not model.has_valid_api_key():
+                raise MissingAPIKeyError(
+                    model_name=str(model.model),
+                    inference_service=model._inference_service_,
+                )
+
     def run(
         self,
         n: int = 1,
@@ -694,91 +852,17 @@ class Jobs(Base):
 
         self.verbose = verbose
 
-        remote_cache = False
-        remote_inference = False
-
-        if not disable_remote_inference:
-            try:
-                coop = Coop()
-                user_edsl_settings = Coop().edsl_settings
-                remote_cache = user_edsl_settings.get("remote_caching", False)
-                remote_inference = user_edsl_settings.get("remote_inference", False)
-            except Exception:
-                pass
-
-        if remote_inference:
-            import time
-            from datetime import datetime
-            from edsl.config import CONFIG
-
-            expected_parrot_url = CONFIG.get("EXPECTED_PARROT_URL")
-
-            self._output("Remote inference activated. Sending job to server...")
-            if remote_cache:
-                self._output(
-                    "Remote caching activated. The remote cache will be used for this job."
-                )
-
-            remote_job_creation_data = coop.remote_inference_create(
-                self,
-                description=remote_inference_description,
-                status="queued",
-                iterations=n,
+        if remote_inference := self.use_remote_inference(disable_remote_inference):
+            remote_job_creation_data = self.create_remote_inference_job(
+                iterations=n, remote_inference_description=remote_inference_description
             )
-            time_queued = datetime.now().strftime("%m/%d/%Y %I:%M:%S %p")
-            job_uuid = remote_job_creation_data.get("uuid")
-            print(f"Remote inference started (Job uuid={job_uuid}).")
-            # print(f"Job queued at {time_queued}.")
-            job_in_queue = True
-            while job_in_queue:
-                remote_job_data = coop.remote_inference_get(job_uuid)
-                status = remote_job_data.get("status")
-                if status == "cancelled":
-                    print("\r" + " " * 80 + "\r", end="")
-                    print("Job cancelled by the user.")
-                    print(
-                        f"See {expected_parrot_url}/home/remote-inference for more details."
-                    )
-                    return None
-                elif status == "failed":
-                    print("\r" + " " * 80 + "\r", end="")
-                    print("Job failed.")
-                    print(
-                        f"See {expected_parrot_url}/home/remote-inference for more details."
-                    )
-                    return None
-                elif status == "completed":
-                    results_uuid = remote_job_data.get("results_uuid")
-                    results = coop.get(results_uuid, expected_object_type="results")
-                    print("\r" + " " * 80 + "\r", end="")
-                    print(
-                        f"Job completed and Results stored on Coop (Results uuid={results_uuid})."
-                    )
-                    return results
-                else:
-                    duration = 5
-                    time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
-                    frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
-                    start_time = time.time()
-                    i = 0
-                    while time.time() - start_time < duration:
-                        print(
-                            f"\r{frames[i % len(frames)]} Job status: {status} - last update: {time_checked}",
-                            end="",
-                            flush=True,
-                        )
-                        time.sleep(0.1)
-                        i += 1
-        else:
-            if check_api_keys:
-                from edsl import Model
+            results = self.poll_remote_inference_job(remote_job_creation_data)
+            if results is None:
+                self._output("Job failed.")
+            return results
 
-                for model in self.models + [Model()]:
-                    if not model.has_valid_api_key():
-                        raise MissingAPIKeyError(
-                            model_name=str(model.model),
-                            inference_service=model._inference_service_,
-                        )
+        if check_api_keys:
+            self.check_api_keys()
 
         # handle cache
         if cache is None or cache is True:
@@ -790,51 +874,14 @@ class Jobs(Base):
 
             cache = Cache()
 
-        if not remote_cache:
-            results = self._run_local(
-                n=n,
-                progress_bar=progress_bar,
-                cache=cache,
-                stop_on_exception=stop_on_exception,
-                sidecar_model=sidecar_model,
-                print_exceptions=print_exceptions,
-                raise_validation_errors=raise_validation_errors,
-            )
-
-            results.cache = cache.new_entries_cache()
-
-            self._output(f"There are {len(cache.keys()):,} entries in the local cache.")
-        else:
-            cache_difference = coop.remote_cache_get_diff(cache.keys())
-
-            client_missing_cacheentries = cache_difference.get(
-                "client_missing_cacheentries", []
-            )
-
-            missing_entry_count = len(client_missing_cacheentries)
-            if missing_entry_count > 0:
-                self._output(
-                    f"Updating local cache with {missing_entry_count:,} new "
-                    f"{'entry' if missing_entry_count == 1 else 'entries'} from remote..."
-                )
-                cache.add_from_dict(
-                    {entry.key: entry for entry in client_missing_cacheentries}
-                )
-                self._output("Local cache updated!")
-            else:
-                self._output("No new entries to add to local cache.")
-
-            server_missing_cacheentry_keys = cache_difference.get(
-                "server_missing_cacheentry_keys", []
-            )
-            server_missing_cacheentries = [
-                entry
-                for key in server_missing_cacheentry_keys
-                if (entry := cache.data.get(key)) is not None
-            ]
-            old_entry_keys = [key for key in cache.keys()]
-
-            self._output("Running job...")
+        remote_cache = self.use_remote_cache()
+        with RemoteCacheSync(
+            coop=Coop(),
+            cache=cache,
+            output_func=self._output,
+            remote_cache=remote_cache,
+            remote_cache_description=remote_cache_description,
+        ) as r:
             results = self._run_local(
                 n=n,
                 progress_bar=progress_bar,
@@ -844,32 +891,8 @@ class Jobs(Base):
                 print_exceptions=print_exceptions,
                 raise_validation_errors=raise_validation_errors,
             )
-            self._output("Job completed!")
-
-            new_cache_entries = list(
-                [entry for entry in cache.values() if entry.key not in old_entry_keys]
-            )
-            server_missing_cacheentries.extend(new_cache_entries)
-
-            new_entry_count = len(server_missing_cacheentries)
-            if new_entry_count > 0:
-                self._output(
-                    f"Updating remote cache with {new_entry_count:,} new "
-                    f"{'entry' if new_entry_count == 1 else 'entries'}..."
-                )
-                coop.remote_cache_create_many(
-                    server_missing_cacheentries,
-                    visibility="private",
-                    description=remote_cache_description,
-                )
-                self._output("Remote cache updated!")
-            else:
-                self._output("No new entries to add to remote cache.")
-
-            results.cache = cache.new_entries_cache()
-
-            self._output(f"There are {len(cache.keys()):,} entries in the local cache.")
 
+        results.cache = cache.new_entries_cache()
         return results
 
     def _run_local(self, *args, **kwargs):

edsl/jobs/interviews/Interview.py

@@ -159,13 +159,13 @@ class Interview:
         return self.task_creators.interview_status
 
     # region: Serialization
-    def _to_dict(self, include_exceptions=False) -> dict[str, Any]:
+    def _to_dict(self, include_exceptions=True) -> dict[str, Any]:
         """Return a dictionary representation of the Interview instance.
         This is just for hashing purposes.
 
         >>> i = Interview.example()
         >>> hash(i)
-        1646262796627658719
+        1217840301076717434
         """
         d = {
             "agent": self.agent._to_dict(),
@@ -177,11 +177,29 @@ class Interview:
         }
         if include_exceptions:
             d["exceptions"] = self.exceptions.to_dict()
+        return d
+    
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "Interview":
+        """Return an Interview instance from a dictionary."""
+        agent = Agent.from_dict(d["agent"])
+        survey = Survey.from_dict(d["survey"])
+        scenario = Scenario.from_dict(d["scenario"])
+        model = LanguageModel.from_dict(d["model"])
+        iteration = d["iteration"]
+        return cls(agent=agent, survey=survey, scenario=scenario, model=model, iteration=iteration)
 
     def __hash__(self) -> int:
         from edsl.utilities.utilities import dict_hash
 
-        return dict_hash(self._to_dict())
+        return dict_hash(self._to_dict(include_exceptions=False))
+    
+    def __eq__(self, other: "Interview") -> bool:
+        """
+        >>> from edsl.jobs.interviews.Interview import Interview; i = Interview.example(); d = i._to_dict(); i2 = Interview.from_dict(d); i == i2
+        True
+        """
+        return hash(self) == hash(other)
 
     # endregion
 

edsl/jobs/interviews/InterviewExceptionCollection.py

@@ -33,6 +33,15 @@ class InterviewExceptionCollection(UserDict):
         """Return the collection of exceptions as a dictionary."""
         newdata = {k: [e.to_dict() for e in v] for k, v in self.data.items()}
         return newdata
+    
+    @classmethod
+    def from_dict(cls, data: dict) -> "InterviewExceptionCollection":
+        """Create an InterviewExceptionCollection from a dictionary."""
+        collection = cls()
+        for question_name, entries in data.items():
+            for entry in entries:
+                collection.add(question_name, InterviewExceptionEntry.from_dict(entry))
+        return collection
 
     def _repr_html_(self) -> str:
         from edsl.utilities.utilities import data_to_html

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -9,7 +9,6 @@ class InterviewExceptionEntry:
         self,
         *,
         exception: Exception,
-        # failed_question: FailedQuestion,
         invigilator: "Invigilator",
         traceback_format="text",
         answers=None,
@@ -133,22 +132,41 @@ class InterviewExceptionEntry:
         )
         console.print(tb)
         return html_output.getvalue()
+    
+    @staticmethod
+    def serialize_exception(exception: Exception) -> dict:
+        return {
+            "type": type(exception).__name__,
+            "message": str(exception),
+            "traceback": "".join(traceback.format_exception(type(exception), exception, exception.__traceback__)),
+        }
+    
+    @staticmethod
+    def deserialize_exception(data: dict) -> Exception:
+        exception_class = globals()[data["type"]]
+        return exception_class(data["message"])
 
     def to_dict(self) -> dict:
         """Return the exception as a dictionary.
 
         >>> entry = InterviewExceptionEntry.example()
-        >>> entry.to_dict()['exception']
-        ValueError()
-
+        >>> _ = entry.to_dict()
         """
         return {
-            "exception": self.exception,
+            "exception": self.serialize_exception(self.exception),
             "time": self.time,
             "traceback": self.traceback,
-            # "failed_question": self.failed_question.to_dict(),
             "invigilator": self.invigilator.to_dict(),
         }
+    
+    @classmethod
+    def from_dict(cls, data: dict) -> "InterviewExceptionEntry":
+        """Create an InterviewExceptionEntry from a dictionary."""
+        from edsl.agents.Invigilator import InvigilatorAI
+
+        exception = cls.deserialize_exception(data["exception"])
+        invigilator = InvigilatorAI.from_dict(data["invigilator"])
+        return cls(exception=exception, invigilator=invigilator)
 
     def push(self):
         from edsl import Coop

edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -1,18 +1,12 @@
 from __future__ import annotations
 import time
-import math
 import asyncio
-import functools
 import threading
 from typing import Coroutine, List, AsyncGenerator, Optional, Union, Generator
 from contextlib import contextmanager
 from collections import UserList
 
-from rich.live import Live
-from rich.console import Console
-
 from edsl.results.Results import Results
-from edsl import shared_globals
 from edsl.jobs.interviews.Interview import Interview
 from edsl.jobs.runners.JobsRunnerStatus import JobsRunnerStatus
 
@@ -25,7 +19,6 @@ from edsl.results.Results import Results
 from edsl.language_models.LanguageModel import LanguageModel
 from edsl.data.Cache import Cache
 
-
 class StatusTracker(UserList):
     def __init__(self, total_tasks: int):
         self.total_tasks = total_tasks
@@ -48,8 +41,6 @@ class JobsRunnerAsyncio:
         self.bucket_collection: "BucketCollection" = jobs.bucket_collection
         self.total_interviews: List["Interview"] = []
 
-        # self.jobs_runner_status = JobsRunnerStatus(self, n=1)
-
     async def run_async_generator(
         self,
         cache: Cache,
@@ -173,19 +164,20 @@ class JobsRunnerAsyncio:
 
         prompt_dictionary = {}
         for answer_key_name in answer_key_names:
-            prompt_dictionary[
-                answer_key_name + "_user_prompt"
-            ] = question_name_to_prompts[answer_key_name]["user_prompt"]
-            prompt_dictionary[
-                answer_key_name + "_system_prompt"
-            ] = question_name_to_prompts[answer_key_name]["system_prompt"]
+            prompt_dictionary[answer_key_name + "_user_prompt"] = (
+                question_name_to_prompts[answer_key_name]["user_prompt"]
+            )
+            prompt_dictionary[answer_key_name + "_system_prompt"] = (
+                question_name_to_prompts[answer_key_name]["system_prompt"]
+            )
 
         raw_model_results_dictionary = {}
+        cache_used_dictionary = {}
         for result in valid_results:
             question_name = result.question_name
-            raw_model_results_dictionary[
-                question_name + "_raw_model_response"
-            ] = result.raw_model_response
+            raw_model_results_dictionary[question_name + "_raw_model_response"] = (
+                result.raw_model_response
+            )
             raw_model_results_dictionary[question_name + "_cost"] = result.cost
             one_use_buys = (
                 "NA"
@@ -195,6 +187,7 @@ class JobsRunnerAsyncio:
                 else 1.0 / result.cost
             )
             raw_model_results_dictionary[question_name + "_one_usd_buys"] = one_use_buys
+            cache_used_dictionary[question_name] = result.cache_used
 
         result = Result(
             agent=interview.agent,
@@ -207,6 +200,7 @@ class JobsRunnerAsyncio:
             survey=interview.survey,
             generated_tokens=generated_tokens_dict,
             comments_dict=comments_dict,
+            cache_used_dict=cache_used_dictionary,
         )
         result.interview_hash = hash(interview)
 
@@ -225,17 +219,16 @@ class JobsRunnerAsyncio:
         }
         interview_hashes = list(interview_lookup.keys())
 
+        task_history = TaskHistory(self.total_interviews, include_traceback=False)
+
         results = Results(
             survey=self.jobs.survey,
             data=sorted(
                 raw_results, key=lambda x: interview_hashes.index(x.interview_hash)
             ),
+            task_history=task_history,
+            cache=cache,
         )
-        results.cache = cache
-        results.task_history = TaskHistory(
-            self.total_interviews, include_traceback=False
-        )
-        results.has_unfixed_exceptions = results.task_history.has_unfixed_exceptions
         results.bucket_collection = self.bucket_collection
 
         if results.has_unfixed_exceptions and print_exceptions:
@@ -263,6 +256,7 @@ class JobsRunnerAsyncio:
             except Exception as e:
                 print(e)
                 remote_logging = False
+
             if remote_logging:
                 filestore = HTMLFileStore(filepath)
                 coop_details = filestore.push(description="Error report")