edsl 0.1.36.dev2__tar.gz → 0.1.36.dev4__tar.gz

{edsl-0.1.36.dev2 → edsl-0.1.36.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edsl
-Version: 0.1.36.dev2
+Version: 0.1.36.dev4
 Summary: Create and analyze LLM-based surveys
 Home-page: https://www.expectedparrot.com/
 License: MIT

edsl-0.1.36.dev4/edsl/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.36.dev4"

{edsl-0.1.36.dev2 → edsl-0.1.36.dev4}/edsl/coop/PriceFetcher.py

@@ -16,30 +16,26 @@ class PriceFetcher:
         if self._cached_prices is not None:
             return self._cached_prices
 
+        import os
         import requests
-        import csv
-        from io import StringIO
-
-        sheet_id = "1SAO3Bhntefl0XQHJv27rMxpvu6uzKDWNXFHRa7jrUDs"
-
-        # Construct the URL to fetch the CSV
-        url = f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv"
+        from edsl import CONFIG
 
         try:
-            # Fetch the CSV data
-            response = requests.get(url)
+            # Fetch the pricing data
+            url = f"{CONFIG.EXPECTED_PARROT_URL}/api/v0/prices"
+            api_key = os.getenv("EXPECTED_PARROT_API_KEY")
+            headers = {}
+            if api_key:
+                headers["Authorization"] = f"Bearer {api_key}"
+            else:
+                headers["Authorization"] = f"Bearer None"
+
+            response = requests.get(url, headers=headers, timeout=20)
             response.raise_for_status()  # Raise an exception for bad responses
 
-            # Parse the CSV data
-            csv_data = StringIO(response.text)
-            reader = csv.reader(csv_data)
-
-            # Convert to list of dictionaries
-            headers = next(reader)
-            data = [dict(zip(headers, row)) for row in reader]
+            # Parse the data
+            data = response.json()
 
-            # self._cached_prices = data
-            # return data
             price_lookup = {}
             for entry in data:
                 service = entry.get("service", None)

{edsl-0.1.36.dev2 → edsl-0.1.36.dev4}/edsl/coop/coop.py

@@ -6,6 +6,7 @@ from typing import Any, Optional, Union, Literal
 from uuid import UUID
 import edsl
 from edsl import CONFIG, CacheEntry, Jobs, Survey
+from edsl.exceptions.coop import CoopNoUUIDError, CoopServerResponseError
 from edsl.coop.utils import (
     EDSLObject,
     ObjectRegistry,
@@ -99,7 +100,7 @@ class Coop:
             if "Authorization" in message:
                 print(message)
                 message = "Please provide an Expected Parrot API key."
-            raise Exception(message)
+            raise CoopServerResponseError(message)
 
     def _json_handle_none(self, value: Any) -> Any:
         """
@@ -116,7 +117,7 @@ class Coop:
         Resolve the uuid from a uuid or a url.
         """
         if not url and not uuid:
-            raise Exception("No uuid or url provided for the object.")
+            raise CoopNoUUIDError("No uuid or url provided for the object.")
         if not uuid and url:
             uuid = url.split("/")[-1]
         return uuid
@@ -584,7 +585,10 @@ class Coop:
         )
         self._resolve_server_response(response)
         response_json = response.json()
-        return response_json.get("cost")
+        return {
+            "credits": response_json.get("cost_in_credits"),
+            "usd": response_json.get("cost_in_usd"),
+        }
 
     ################
     # Remote Errors

edsl-0.1.36.dev4/edsl/data/RemoteCacheSync.py

@@ -0,0 +1,84 @@
+class RemoteCacheSync:
+    def __init__(self, coop, cache, output_func, remote_cache=True, remote_cache_description=""):
+        self.coop = coop
+        self.cache = cache
+        self._output = output_func
+        self.remote_cache = remote_cache
+        self.old_entry_keys = []
+        self.new_cache_entries = []
+        self.remote_cache_description = remote_cache_description
+
+    def __enter__(self):
+        if self.remote_cache:
+            self._sync_from_remote()
+            self.old_entry_keys = list(self.cache.keys())
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.remote_cache:
+            self._sync_to_remote()
+        return False  # Propagate exceptions
+
+    def _sync_from_remote(self):
+        cache_difference = self.coop.remote_cache_get_diff(self.cache.keys())
+        client_missing_cacheentries = cache_difference.get("client_missing_cacheentries", [])
+        missing_entry_count = len(client_missing_cacheentries)
+        
+        if missing_entry_count > 0:
+            self._output(
+                f"Updating local cache with {missing_entry_count:,} new "
+                f"{'entry' if missing_entry_count == 1 else 'entries'} from remote..."
+            )
+            self.cache.add_from_dict({entry.key: entry for entry in client_missing_cacheentries})
+            self._output("Local cache updated!")
+        else:
+            self._output("No new entries to add to local cache.")
+
+    def _sync_to_remote(self):
+        cache_difference = self.coop.remote_cache_get_diff(self.cache.keys())
+        server_missing_cacheentry_keys = cache_difference.get("server_missing_cacheentry_keys", [])
+        server_missing_cacheentries = [
+            entry
+            for key in server_missing_cacheentry_keys
+            if (entry := self.cache.data.get(key)) is not None
+        ]
+        
+        new_cache_entries = [
+            entry for entry in self.cache.values() if entry.key not in self.old_entry_keys
+        ]
+        server_missing_cacheentries.extend(new_cache_entries)
+        new_entry_count = len(server_missing_cacheentries)
+        
+        if new_entry_count > 0:
+            self._output(
+                f"Updating remote cache with {new_entry_count:,} new "
+                f"{'entry' if new_entry_count == 1 else 'entries'}..."
+            )
+            self.coop.remote_cache_create_many(
+                server_missing_cacheentries,
+                visibility="private",
+                description=self.remote_cache_description,
+            )
+            self._output("Remote cache updated!")
+        else:
+            self._output("No new entries to add to remote cache.")
+        
+        self._output(f"There are {len(self.cache.keys()):,} entries in the local cache.")
+
+# # Usage example
+# def run_job(self, n, progress_bar, cache, stop_on_exception, sidecar_model, print_exceptions, raise_validation_errors, use_remote_cache=True):
+#     with RemoteCacheSync(self.coop, cache, self._output, remote_cache=use_remote_cache):
+#         self._output("Running job...")
+#         results = self._run_local(
+#             n=n,
+#             progress_bar=progress_bar,
+#             cache=cache,
+#             stop_on_exception=stop_on_exception,
+#             sidecar_model=sidecar_model,
+#             print_exceptions=print_exceptions,
+#             raise_validation_errors=raise_validation_errors,
+#         )
+#         self._output("Job completed!")
+    
+#     results.cache = cache.new_entries_cache()
+#     return results

edsl-0.1.36.dev4/edsl/exceptions/coop.py

@@ -0,0 +1,10 @@
+class CoopErrors(Exception):
+    pass
+
+
+class CoopNoUUIDError(CoopErrors):
+    pass
+
+
+class CoopServerResponseError(CoopErrors):
+    pass

{edsl-0.1.36.dev2 → edsl-0.1.36.dev4}/edsl/jobs/Jobs.py

@@ -1,8 +1,10 @@
 # """The Jobs class is a collection of agents, scenarios and models and one survey."""
 from __future__ import annotations
 import warnings
+import requests
 from itertools import product
 from typing import Optional, Union, Sequence, Generator
+
 from edsl.Base import Base
 from edsl.exceptions import MissingAPIKeyError
 from edsl.jobs.buckets.BucketCollection import BucketCollection
@@ -10,6 +12,9 @@ from edsl.jobs.interviews.Interview import Interview
 from edsl.jobs.runners.JobsRunnerAsyncio import JobsRunnerAsyncio
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
+from edsl.data.RemoteCacheSync import RemoteCacheSync
+from edsl.exceptions.coop import CoopServerResponseError
+
 
 class Jobs(Base):
     """
@@ -203,10 +208,6 @@ class Jobs(Base):
             ]
         )
         return d
-        # if table:
-        #     d.to_scenario_list().print(format="rich")
-        # else:
-        #     return d
 
     def show_prompts(self) -> None:
         """Print the prompts."""
@@ -219,11 +220,11 @@ class Jobs(Base):
         price_lookup: dict,
         inference_service: str,
         model: str,
-    ):
+    ) -> dict:
         """Estimates the cost of a prompt. Takes piping into account."""
 
         def get_piping_multiplier(prompt: str):
-            """Returns 2 if a prompt includes Jinja brances, and 1 otherwise."""
+            """Returns 2 if a prompt includes Jinja braces, and 1 otherwise."""
 
             if "{{" in prompt and "}}" in prompt:
                 return 2
@@ -231,9 +232,25 @@ class Jobs(Base):
 
         # Look up prices per token
         key = (inference_service, model)
-        relevant_prices = price_lookup[key]
-        output_price_per_token = 1 / float(relevant_prices["output"]["one_usd_buys"])
-        input_price_per_token = 1 / float(relevant_prices["input"]["one_usd_buys"])
+
+        try:
+            relevant_prices = price_lookup[key]
+            output_price_per_token = 1 / float(
+                relevant_prices["output"]["one_usd_buys"]
+            )
+            input_price_per_token = 1 / float(relevant_prices["input"]["one_usd_buys"])
+        except KeyError:
+            # A KeyError is likely to occur if we cannot retrieve prices (the price_lookup dict is empty)
+            # Use a sensible default
+
+            import warnings
+
+            warnings.warn(
+                "Price data could not be retrieved. Using default estimates for input and output token prices. Input: $0.15 / 1M tokens; Output: $0.60 / 1M tokens"
+            )
+
+            output_price_per_token = 0.00000015  # $0.15 / 1M tokens
+            input_price_per_token = 0.00000060  # $0.60 / 1M tokens
 
         # Compute the number of characters (double if the question involves piping)
         user_prompt_chars = len(str(user_prompt)) * get_piping_multiplier(
@@ -258,7 +275,7 @@ class Jobs(Base):
             "cost": cost,
         }
 
-    def estimate_job_cost_from_external_prices(self, price_lookup: dict):
+    def estimate_job_cost_from_external_prices(self, price_lookup: dict) -> dict:
         """
         Estimates the cost of a job according to the following assumptions:
 
@@ -341,7 +358,7 @@ class Jobs(Base):
 
         return output
 
-    def estimate_job_cost(self):
+    def estimate_job_cost(self) -> dict:
         """
         Estimates the cost of a job according to the following assumptions:
 
@@ -357,6 +374,25 @@ class Jobs(Base):
 
         return self.estimate_job_cost_from_external_prices(price_lookup=price_lookup)
 
+    @staticmethod
+    def compute_job_cost(job_results: "Results") -> float:
+        """
+        Computes the cost of a completed job in USD.
+        """
+        total_cost = 0
+        for result in job_results:
+            for key in result.raw_model_response:
+                if key.endswith("_cost"):
+                    result_cost = result.raw_model_response[key]
+
+                    question_name = key.removesuffix("_cost")
+                    cache_used = result.cache_used_dict[question_name]
+
+                    if isinstance(result_cost, (int, float)) and not cache_used:
+                        total_cost += result_cost
+
+        return total_cost
+
     @staticmethod
     def _get_container_class(object):
         from edsl.agents.AgentList import AgentList
@@ -580,7 +616,7 @@ class Jobs(Base):
 
     def _output(self, message) -> None:
         """Check if a Job is verbose. If so, print the message."""
-        if self.verbose:
+        if hasattr(self, "verbose") and self.verbose:
             print(message)
 
     def _check_parameters(self, strict=False, warn=False) -> None:
@@ -657,6 +693,122 @@ class Jobs(Base):
             return False
         return self._raise_validation_errors
 
+    def create_remote_inference_job(
+        self, iterations: int = 1, remote_inference_description: Optional[str] = None
+    ):
+        """ """
+        from edsl.coop.coop import Coop
+
+        coop = Coop()
+        self._output("Remote inference activated. Sending job to server...")
+        remote_job_creation_data = coop.remote_inference_create(
+            self,
+            description=remote_inference_description,
+            status="queued",
+            iterations=iterations,
+        )
+        return remote_job_creation_data
+
+    @staticmethod
+    def check_status(job_uuid):
+        from edsl.coop.coop import Coop
+
+        coop = Coop()
+        return coop.remote_inference_get(job_uuid)
+
+    def poll_remote_inference_job(
+        self, remote_job_creation_data: dict
+    ) -> Union[Results, None]:
+        from edsl.coop.coop import Coop
+        import time
+        from datetime import datetime
+        from edsl.config import CONFIG
+
+        expected_parrot_url = CONFIG.get("EXPECTED_PARROT_URL")
+
+        job_uuid = remote_job_creation_data.get("uuid")
+
+        coop = Coop()
+        job_in_queue = True
+        while job_in_queue:
+            remote_job_data = coop.remote_inference_get(job_uuid)
+            status = remote_job_data.get("status")
+            if status == "cancelled":
+                print("\r" + " " * 80 + "\r", end="")
+                print("Job cancelled by the user.")
+                print(
+                    f"See {expected_parrot_url}/home/remote-inference for more details."
+                )
+                return None
+            elif status == "failed":
+                print("\r" + " " * 80 + "\r", end="")
+                print("Job failed.")
+                print(
+                    f"See {expected_parrot_url}/home/remote-inference for more details."
+                )
+                return None
+            elif status == "completed":
+                results_uuid = remote_job_data.get("results_uuid")
+                results = coop.get(results_uuid, expected_object_type="results")
+                print("\r" + " " * 80 + "\r", end="")
+                print(
+                    f"Job completed and Results stored on Coop (Results uuid={results_uuid})."
+                )
+                return results
+            else:
+                duration = 5
+                time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
+                frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
+                start_time = time.time()
+                i = 0
+                while time.time() - start_time < duration:
+                    print(
+                        f"\r{frames[i % len(frames)]} Job status: {status} - last update: {time_checked}",
+                        end="",
+                        flush=True,
+                    )
+                    time.sleep(0.1)
+                    i += 1
+
+    def use_remote_inference(self, disable_remote_inference: bool):
+        if disable_remote_inference:
+            return False
+        if not disable_remote_inference:
+            try:
+                from edsl import Coop
+
+                user_edsl_settings = Coop().edsl_settings
+                return user_edsl_settings.get("remote_inference", False)
+            except requests.ConnectionError:
+                pass
+            except CoopServerResponseError as e:
+                pass
+
+        return False
+
+    def use_remote_cache(self):
+        try:
+            from edsl import Coop
+
+            user_edsl_settings = Coop().edsl_settings
+            return user_edsl_settings.get("remote_caching", False)
+        except requests.ConnectionError:
+            pass
+        except CoopServerResponseError as e:
+            pass
+
+        return False
+
+    def check_api_keys(self):
+        from edsl import Model
+
+        for model in self.models + [Model()]:
+            if not model.has_valid_api_key():
+                raise MissingAPIKeyError(
+                    model_name=str(model.model),
+                    inference_service=model._inference_service_,
+                )
+
     def run(
         self,
         n: int = 1,
@@ -694,91 +846,17 @@ class Jobs(Base):
 
         self.verbose = verbose
 
-        remote_cache = False
-        remote_inference = False
-
-        if not disable_remote_inference:
-            try:
-                coop = Coop()
-                user_edsl_settings = Coop().edsl_settings
-                remote_cache = user_edsl_settings.get("remote_caching", False)
-                remote_inference = user_edsl_settings.get("remote_inference", False)
-            except Exception:
-                pass
-
-        if remote_inference:
-            import time
-            from datetime import datetime
-            from edsl.config import CONFIG
-
-            expected_parrot_url = CONFIG.get("EXPECTED_PARROT_URL")
-
-            self._output("Remote inference activated. Sending job to server...")
-            if remote_cache:
-                self._output(
-                    "Remote caching activated. The remote cache will be used for this job."
-                )
-
-            remote_job_creation_data = coop.remote_inference_create(
-                self,
-                description=remote_inference_description,
-                status="queued",
-                iterations=n,
+        if remote_inference := self.use_remote_inference(disable_remote_inference):
+            remote_job_creation_data = self.create_remote_inference_job(
+                iterations=n, remote_inference_description=remote_inference_description
             )
-            time_queued = datetime.now().strftime("%m/%d/%Y %I:%M:%S %p")
-            job_uuid = remote_job_creation_data.get("uuid")
-            print(f"Remote inference started (Job uuid={job_uuid}).")
-            # print(f"Job queued at {time_queued}.")
-            job_in_queue = True
-            while job_in_queue:
-                remote_job_data = coop.remote_inference_get(job_uuid)
-                status = remote_job_data.get("status")
-                if status == "cancelled":
-                    print("\r" + " " * 80 + "\r", end="")
-                    print("Job cancelled by the user.")
-                    print(
-                        f"See {expected_parrot_url}/home/remote-inference for more details."
-                    )
-                    return None
-                elif status == "failed":
-                    print("\r" + " " * 80 + "\r", end="")
-                    print("Job failed.")
-                    print(
-                        f"See {expected_parrot_url}/home/remote-inference for more details."
-                    )
-                    return None
-                elif status == "completed":
-                    results_uuid = remote_job_data.get("results_uuid")
-                    results = coop.get(results_uuid, expected_object_type="results")
-                    print("\r" + " " * 80 + "\r", end="")
-                    print(
-                        f"Job completed and Results stored on Coop (Results uuid={results_uuid})."
-                    )
-                    return results
-                else:
-                    duration = 5
-                    time_checked = datetime.now().strftime("%Y-%m-%d %I:%M:%S %p")
-                    frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
-                    start_time = time.time()
-                    i = 0
-                    while time.time() - start_time < duration:
-                        print(
-                            f"\r{frames[i % len(frames)]} Job status: {status} - last update: {time_checked}",
-                            end="",
-                            flush=True,
-                        )
-                        time.sleep(0.1)
-                        i += 1
-        else:
-            if check_api_keys:
-                from edsl import Model
+            results = self.poll_remote_inference_job(remote_job_creation_data)
+            if results is None:
+                self._output("Job failed.")
+            return results
 
-                for model in self.models + [Model()]:
-                    if not model.has_valid_api_key():
-                        raise MissingAPIKeyError(
-                            model_name=str(model.model),
-                            inference_service=model._inference_service_,
-                        )
+        if check_api_keys:
+            self.check_api_keys()
 
         # handle cache
         if cache is None or cache is True:
@@ -790,51 +868,14 @@ class Jobs(Base):
 
             cache = Cache()
 
-        if not remote_cache:
-            results = self._run_local(
-                n=n,
-                progress_bar=progress_bar,
-                cache=cache,
-                stop_on_exception=stop_on_exception,
-                sidecar_model=sidecar_model,
-                print_exceptions=print_exceptions,
-                raise_validation_errors=raise_validation_errors,
-            )
-
-            results.cache = cache.new_entries_cache()
-
-            self._output(f"There are {len(cache.keys()):,} entries in the local cache.")
-        else:
-            cache_difference = coop.remote_cache_get_diff(cache.keys())
-
-            client_missing_cacheentries = cache_difference.get(
-                "client_missing_cacheentries", []
-            )
-
-            missing_entry_count = len(client_missing_cacheentries)
-            if missing_entry_count > 0:
-                self._output(
-                    f"Updating local cache with {missing_entry_count:,} new "
-                    f"{'entry' if missing_entry_count == 1 else 'entries'} from remote..."
-                )
-                cache.add_from_dict(
-                    {entry.key: entry for entry in client_missing_cacheentries}
-                )
-                self._output("Local cache updated!")
-            else:
-                self._output("No new entries to add to local cache.")
-
-            server_missing_cacheentry_keys = cache_difference.get(
-                "server_missing_cacheentry_keys", []
-            )
-            server_missing_cacheentries = [
-                entry
-                for key in server_missing_cacheentry_keys
-                if (entry := cache.data.get(key)) is not None
-            ]
-            old_entry_keys = [key for key in cache.keys()]
-
-            self._output("Running job...")
+        remote_cache = self.use_remote_cache()
+        with RemoteCacheSync(
+            coop=Coop(),
+            cache=cache,
+            output_func=self._output,
+            remote_cache=remote_cache,
+            remote_cache_description=remote_cache_description,
+        ) as r:
             results = self._run_local(
                 n=n,
                 progress_bar=progress_bar,
@@ -844,32 +885,8 @@ class Jobs(Base):
                 print_exceptions=print_exceptions,
                 raise_validation_errors=raise_validation_errors,
             )
-            self._output("Job completed!")
-
-            new_cache_entries = list(
-                [entry for entry in cache.values() if entry.key not in old_entry_keys]
-            )
-            server_missing_cacheentries.extend(new_cache_entries)
-
-            new_entry_count = len(server_missing_cacheentries)
-            if new_entry_count > 0:
-                self._output(
-                    f"Updating remote cache with {new_entry_count:,} new "
-                    f"{'entry' if new_entry_count == 1 else 'entries'}..."
-                )
-                coop.remote_cache_create_many(
-                    server_missing_cacheentries,
-                    visibility="private",
-                    description=remote_cache_description,
-                )
-                self._output("Remote cache updated!")
-            else:
-                self._output("No new entries to add to remote cache.")
-
-            results.cache = cache.new_entries_cache()
-
-            self._output(f"There are {len(cache.keys()):,} entries in the local cache.")
 
+        results.cache = cache.new_entries_cache()
         return results
 
     def _run_local(self, *args, **kwargs):

{edsl-0.1.36.dev2 → edsl-0.1.36.dev4}/edsl/jobs/interviews/Interview.py

@@ -159,13 +159,13 @@ class Interview:
         return self.task_creators.interview_status
 
     # region: Serialization
-    def _to_dict(self, include_exceptions=False) -> dict[str, Any]:
+    def _to_dict(self, include_exceptions=True) -> dict[str, Any]:
         """Return a dictionary representation of the Interview instance.
         This is just for hashing purposes.
 
         >>> i = Interview.example()
         >>> hash(i)
-        1646262796627658719
+        1217840301076717434
         """
         d = {
             "agent": self.agent._to_dict(),
@@ -177,11 +177,29 @@ class Interview:
         }
         if include_exceptions:
             d["exceptions"] = self.exceptions.to_dict()
+        return d
+    
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> "Interview":
+        """Return an Interview instance from a dictionary."""
+        agent = Agent.from_dict(d["agent"])
+        survey = Survey.from_dict(d["survey"])
+        scenario = Scenario.from_dict(d["scenario"])
+        model = LanguageModel.from_dict(d["model"])
+        iteration = d["iteration"]
+        return cls(agent=agent, survey=survey, scenario=scenario, model=model, iteration=iteration)
 
     def __hash__(self) -> int:
         from edsl.utilities.utilities import dict_hash
 
-        return dict_hash(self._to_dict())
+        return dict_hash(self._to_dict(include_exceptions=False))
+    
+    def __eq__(self, other: "Interview") -> bool:
+        """
+        >>> from edsl.jobs.interviews.Interview import Interview; i = Interview.example(); d = i._to_dict(); i2 = Interview.from_dict(d); i == i2
+        True
+        """
+        return hash(self) == hash(other)
 
     # endregion