edsl 0.1.29.dev3__py3-none-any.whl → 0.1.30__py3-none-any.whl

edsl/data/Cache.py

@@ -7,17 +7,10 @@ import json
 import os
 import warnings
 from typing import Optional, Union
-
-from edsl.config import CONFIG
-from edsl.data.CacheEntry import CacheEntry
-from edsl.data.SQLiteDict import SQLiteDict
 from edsl.Base import Base
+from edsl.data.CacheEntry import CacheEntry
 from edsl.utilities.utilities import dict_hash
-
-from edsl.utilities.decorators import (
-    add_edsl_version,
-    remove_edsl_version,
-)
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
 
 class Cache(Base):
@@ -38,9 +31,10 @@ class Cache(Base):
         self,
         *,
         filename: Optional[str] = None,
-        data: Optional[Union[SQLiteDict, dict]] = None,
+        data: Optional[Union["SQLiteDict", dict]] = None,
         immediate_write: bool = True,
         method=None,
+        verbose=False,
     ):
         """
         Create two dictionaries to store the cache data.
@@ -59,6 +53,7 @@ class Cache(Base):
         self.new_entries = {}
         self.new_entries_to_write_later = {}
         self.coop = None
+        self.verbose = verbose
 
         self.filename = filename
         if filename and data:
@@ -104,6 +99,8 @@ class Cache(Base):
 
     def _perform_checks(self):
         """Perform checks on the cache."""
+        from edsl.data.CacheEntry import CacheEntry
+
         if any(not isinstance(value, CacheEntry) for value in self.data.values()):
             raise Exception("Not all values are CacheEntry instances")
         if self.method is not None:
@@ -120,7 +117,7 @@ class Cache(Base):
         system_prompt: str,
         user_prompt: str,
         iteration: int,
-    ) -> Union[None, str]:
+    ) -> tuple(Union[None, str], str):
         """
         Fetch a value (LLM output) from the cache.
 
@@ -133,11 +130,13 @@ class Cache(Base):
         Return None if the response is not found.
 
         >>> c = Cache()
-        >>> c.fetch(model="gpt-3", parameters="default", system_prompt="Hello", user_prompt="Hi", iteration=1) is None
+        >>> c.fetch(model="gpt-3", parameters="default", system_prompt="Hello", user_prompt="Hi", iteration=1)[0] is None
         True
 
 
         """
+        from edsl.data.CacheEntry import CacheEntry
+
         key = CacheEntry.gen_key(
             model=model,
             parameters=parameters,
@@ -147,8 +146,13 @@ class Cache(Base):
         )
         entry = self.data.get(key, None)
         if entry is not None:
+            if self.verbose:
+                print(f"Cache hit for key: {key}")
             self.fetched_data[key] = entry
-        return None if entry is None else entry.output
+        else:
+            if self.verbose:
+                print(f"Cache miss for key: {key}")
+        return None if entry is None else entry.output, key
 
     def store(
         self,
@@ -171,6 +175,7 @@ class Cache(Base):
         * If `immediate_write` is True , the key-value pair is added to `self.data`
         * If `immediate_write` is False, the key-value pair is added to `self.new_entries_to_write_later`
         """
+
         entry = CacheEntry(
             model=model,
             parameters=parameters,
@@ -188,13 +193,14 @@ class Cache(Base):
         return key
 
     def add_from_dict(
-        self, new_data: dict[str, CacheEntry], write_now: Optional[bool] = True
+        self, new_data: dict[str, "CacheEntry"], write_now: Optional[bool] = True
     ) -> None:
         """
         Add entries to the cache from a dictionary.
 
         :param write_now: Whether to write to the cache immediately (similar to `immediate_write`).
         """
+
         for key, value in new_data.items():
             if key in self.data:
                 if value != self.data[key]:
@@ -231,6 +237,8 @@ class Cache(Base):
 
         :param write_now: Whether to write to the cache immediately (similar to `immediate_write`).
         """
+        from edsl.data.SQLiteDict import SQLiteDict
+
         db = SQLiteDict(db_path)
         new_data = {}
         for key, value in db.items():
@@ -242,6 +250,8 @@ class Cache(Base):
         """
         Construct a Cache from a SQLite database.
         """
+        from edsl.data.SQLiteDict import SQLiteDict
+
         return cls(data=SQLiteDict(db_path))
 
     @classmethod
@@ -268,6 +278,8 @@ class Cache(Base):
         * If `db_path` is provided, the cache will be stored in an SQLite database.
         """
         # if a file doesn't exist at jsonfile, throw an error
+        from edsl.data.SQLiteDict import SQLiteDict
+
         if not os.path.exists(jsonlfile):
             raise FileNotFoundError(f"File {jsonlfile} not found")
 
@@ -286,6 +298,8 @@ class Cache(Base):
         """
         ## TODO: Check to make sure not over-writing (?)
         ## Should be added to SQLiteDict constructor (?)
+        from edsl.data.SQLiteDict import SQLiteDict
+
         new_data = SQLiteDict(db_path)
         for key, value in self.data.items():
             new_data[key] = value
@@ -340,6 +354,9 @@ class Cache(Base):
         for key, entry in self.new_entries_to_write_later.items():
             self.data[key] = entry
 
+        if self.filename:
+            self.write(self.filename)
+
     ####################
     # DUNDER / USEFUL
     ####################
@@ -456,12 +473,18 @@ class Cache(Base):
         webbrowser.open("file://" + filepath)
 
     @classmethod
-    def example(cls) -> Cache:
+    def example(cls, randomize: bool = False) -> Cache:
         """
-        Return an example Cache.
-        The example Cache has one entry.
+        Returns an example Cache instance.
+
+        :param randomize: If True, uses CacheEntry's randomize method.
         """
-        return cls(data={CacheEntry.example().key: CacheEntry.example()})
+        return cls(
+            data={
+                CacheEntry.example(randomize).key: CacheEntry.example(),
+                CacheEntry.example(randomize).key: CacheEntry.example(),
+            }
+        )
 
 
 if __name__ == "__main__":

edsl/data/CacheEntry.py

@@ -2,11 +2,8 @@ from __future__ import annotations
 import json
 import datetime
 import hashlib
-import random
 from typing import Optional
-
-
-# TODO: Timestamp should probably be float?
+from uuid import uuid4
 
 
 class CacheEntry:
@@ -151,10 +148,12 @@ class CacheEntry:
     @classmethod
     def example(cls, randomize: bool = False) -> CacheEntry:
         """
-        Returns a CacheEntry example.
+        Returns an example CacheEntry instance.
+
+        :param randomize: If True, adds a random string to the system prompt.
         """
-        # if random, create a random number for 0-100
-        addition = "" if not randomize else str(random.randint(0, 1000))
+        # if random, create a uuid
+        addition = "" if not randomize else str(uuid4())
         return CacheEntry(
             model="gpt-3.5-turbo",
             parameters={"temperature": 0.5},

edsl/data/SQLiteDict.py

@@ -1,9 +1,7 @@
 from __future__ import annotations
 import json
-from sqlalchemy import create_engine
-from sqlalchemy.exc import SQLAlchemyError
-from sqlalchemy.orm import sessionmaker
 from typing import Any, Generator, Optional, Union
+
 from edsl.config import CONFIG
 from edsl.data.CacheEntry import CacheEntry
 from edsl.data.orm import Base, Data
@@ -25,10 +23,16 @@ class SQLiteDict:
         >>> import os; os.unlink(temp_db_path)  # Clean up the temp file after the test
 
         """
+        from sqlalchemy.exc import SQLAlchemyError
+        from sqlalchemy.orm import sessionmaker
+        from sqlalchemy import create_engine
+
         self.db_path = db_path or CONFIG.get("EDSL_DATABASE_PATH")
         if not self.db_path.startswith("sqlite:///"):
             self.db_path = f"sqlite:///{self.db_path}"
         try:
+            from edsl.data.orm import Base, Data
+
             self.engine = create_engine(self.db_path, echo=False, future=True)
             Base.metadata.create_all(self.engine)
             self.Session = sessionmaker(bind=self.engine)
@@ -55,6 +59,8 @@ class SQLiteDict:
         if not isinstance(value, CacheEntry):
             raise ValueError(f"Value must be a CacheEntry object (got {type(value)}).")
         with self.Session() as db:
+            from edsl.data.orm import Base, Data
+
             db.merge(Data(key=key, value=json.dumps(value.to_dict())))
             db.commit()
 
@@ -69,6 +75,8 @@ class SQLiteDict:
         True
         """
         with self.Session() as db:
+            from edsl.data.orm import Base, Data
+
             value = db.query(Data).filter_by(key=key).first()
             if not value:
                 raise KeyError(f"Key '{key}' not found.")

edsl/data_transfer_models.py

@@ -17,6 +17,8 @@ class AgentResponseDict(UserDict):
         cached_response=None,
         raw_model_response=None,
         simple_model_raw_response=None,
+        cache_used=None,
+        cache_key=None,
     ):
         """Initialize the AgentResponseDict object."""
         usage = usage or {"prompt_tokens": 0, "completion_tokens": 0}
@@ -30,5 +32,7 @@ class AgentResponseDict(UserDict):
                 "cached_response": cached_response,
                 "raw_model_response": raw_model_response,
                 "simple_model_raw_response": simple_model_raw_response,
+                "cache_used": cache_used,
+                "cache_key": cache_key,
             }
         )

edsl/jobs/Answers.py

@@ -8,7 +8,15 @@ class Answers(UserDict):
     """Helper class to hold the answers to a survey."""
 
     def add_answer(self, response, question) -> None:
-        """Add a response to the answers dictionary."""
+        """Add a response to the answers dictionary.
+
+        >>> from edsl import QuestionFreeText
+        >>> q = QuestionFreeText.example()
+        >>> answers = Answers()
+        >>> answers.add_answer({"answer": "yes"}, q)
+        >>> answers[q.question_name]
+        'yes'
+        """
         answer = response.get("answer")
         comment = response.pop("comment", None)
         # record the answer
@@ -42,3 +50,9 @@ class Answers(UserDict):
             table.add_row(attr_name, repr(attr_value))
 
         return table
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

edsl/jobs/Jobs.py

@@ -1,30 +1,15 @@
 # """The Jobs class is a collection of agents, scenarios and models and one survey."""
 from __future__ import annotations
-import os
 import warnings
 from itertools import product
 from typing import Optional, Union, Sequence, Generator
-from edsl import Model
-from edsl.agents import Agent
-from edsl.agents.AgentList import AgentList
+
 from edsl.Base import Base
-from edsl.data.Cache import Cache
-from edsl.data.CacheHandler import CacheHandler
-from edsl.results.Dataset import Dataset
 
-from edsl.exceptions.jobs import MissingRemoteInferenceError
 from edsl.exceptions import MissingAPIKeyError
 from edsl.jobs.buckets.BucketCollection import BucketCollection
 from edsl.jobs.interviews.Interview import Interview
-from edsl.language_models import LanguageModel
-from edsl.results import Results
-from edsl.scenarios import Scenario
-from edsl import ScenarioList
-from edsl.surveys import Survey
 from edsl.jobs.runners.JobsRunnerAsyncio import JobsRunnerAsyncio
-
-from edsl.language_models.ModelList import ModelList
-
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
 
@@ -37,10 +22,10 @@ class Jobs(Base):
 
     def __init__(
         self,
-        survey: Survey,
-        agents: Optional[list[Agent]] = None,
-        models: Optional[list[LanguageModel]] = None,
-        scenarios: Optional[list[Scenario]] = None,
+        survey: "Survey",
+        agents: Optional[list["Agent"]] = None,
+        models: Optional[list["LanguageModel"]] = None,
+        scenarios: Optional[list["Scenario"]] = None,
     ):
         """Initialize a Jobs instance.
 
@@ -50,8 +35,8 @@ class Jobs(Base):
         :param scenarios: a list of scenarios
         """
         self.survey = survey
-        self.agents: AgentList = agents
-        self.scenarios: ScenarioList = scenarios
+        self.agents: "AgentList" = agents
+        self.scenarios: "ScenarioList" = scenarios
         self.models = models
 
         self.__bucket_collection = None
@@ -62,6 +47,8 @@ class Jobs(Base):
 
     @models.setter
     def models(self, value):
+        from edsl import ModelList
+
         if value:
             if not isinstance(value, ModelList):
                 self._models = ModelList(value)
@@ -76,6 +63,8 @@ class Jobs(Base):
 
     @agents.setter
     def agents(self, value):
+        from edsl import AgentList
+
         if value:
             if not isinstance(value, AgentList):
                 self._agents = AgentList(value)
@@ -90,6 +79,8 @@ class Jobs(Base):
 
     @scenarios.setter
     def scenarios(self, value):
+        from edsl import ScenarioList
+
         if value:
             if not isinstance(value, ScenarioList):
                 self._scenarios = ScenarioList(value)
@@ -101,10 +92,10 @@ class Jobs(Base):
     def by(
         self,
         *args: Union[
-            Agent,
-            Scenario,
-            LanguageModel,
-            Sequence[Union[Agent, Scenario, LanguageModel]],
+            "Agent",
+            "Scenario",
+            "LanguageModel",
+            Sequence[Union["Agent", "Scenario", "LanguageModel"]],
         ],
     ) -> Jobs:
         """
@@ -144,7 +135,7 @@ class Jobs(Base):
         setattr(self, objects_key, new_objects)  # update the job
         return self
 
-    def prompts(self) -> Dataset:
+    def prompts(self) -> "Dataset":
         """Return a Dataset of prompts that will be used.
 
 
@@ -160,6 +151,7 @@ class Jobs(Base):
         user_prompts = []
         system_prompts = []
         scenario_indices = []
+        from edsl.results.Dataset import Dataset
 
         for interview_index, interview in enumerate(interviews):
             invigilators = list(interview._build_invigilators(debug=False))
@@ -182,7 +174,10 @@ class Jobs(Base):
 
     @staticmethod
     def _get_container_class(object):
-        from edsl import AgentList
+        from edsl.agents.AgentList import AgentList
+        from edsl.agents.Agent import Agent
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.scenarios.ScenarioList import ScenarioList
 
         if isinstance(object, Agent):
             return AgentList
@@ -218,6 +213,10 @@ class Jobs(Base):
     def _get_current_objects_of_this_type(
         self, object: Union[Agent, Scenario, LanguageModel]
     ) -> tuple[list, str]:
+        from edsl.agents.Agent import Agent
+        from edsl.scenarios.Scenario import Scenario
+        from edsl.language_models.LanguageModel import LanguageModel
+
         """Return the current objects of the same type as the first argument.
 
         >>> from edsl.jobs import Jobs
@@ -246,6 +245,9 @@ class Jobs(Base):
     @staticmethod
     def _get_empty_container_object(object):
         from edsl import AgentList
+        from edsl import Agent
+        from edsl import Scenario
+        from edsl import ScenarioList
 
         if isinstance(object, Agent):
             return AgentList([])
@@ -310,12 +312,12 @@ class Jobs(Base):
         with us filling in defaults.
         """
         # if no agents, models, or scenarios are set, set them to defaults
+        from edsl.agents.Agent import Agent
+        from edsl.language_models.registry import Model
+        from edsl.scenarios.Scenario import Scenario
+
         self.agents = self.agents or [Agent()]
         self.models = self.models or [Model()]
-        # if remote, set all the models to remote
-        if hasattr(self, "remote") and self.remote:
-            for model in self.models:
-                model.remote = True
         self.scenarios = self.scenarios or [Scenario()]
         for agent, scenario, model in product(self.agents, self.scenarios, self.models):
             yield Interview(
@@ -329,6 +331,7 @@ class Jobs(Base):
         These buckets are used to track API calls and token usage.
 
         >>> from edsl.jobs import Jobs
+        >>> from edsl import Model
         >>> j = Jobs.example().by(Model(temperature = 1), Model(temperature = 0.5))
         >>> bc = j.create_bucket_collection()
         >>> bc
@@ -368,14 +371,16 @@ class Jobs(Base):
         if self.verbose:
             print(message)
 
-    def _check_parameters(self, strict=False, warn = True) -> None:
+    def _check_parameters(self, strict=False, warn=False) -> None:
         """Check if the parameters in the survey and scenarios are consistent.
 
         >>> from edsl import QuestionFreeText
+        >>> from edsl import Survey
+        >>> from edsl import Scenario
         >>> q = QuestionFreeText(question_text = "{{poo}}", question_name = "ugly_question")
         >>> j = Jobs(survey = Survey(questions=[q]))
         >>> with warnings.catch_warnings(record=True) as w:
-        ...     j._check_parameters()
+        ...     j._check_parameters(warn = True)
         ...     assert len(w) == 1
         ...     assert issubclass(w[-1].category, UserWarning)
         ...     assert "The following parameters are in the survey but not in the scenarios" in str(w[-1].message)
@@ -413,15 +418,13 @@ class Jobs(Base):
         progress_bar: bool = False,
         stop_on_exception: bool = False,
         cache: Union[Cache, bool] = None,
-        remote: bool = (
-            False if os.getenv("DEFAULT_RUN_MODE", "local") == "local" else True
-        ),
         check_api_keys: bool = False,
         sidecar_model: Optional[LanguageModel] = None,
         batch_mode: Optional[bool] = None,
         verbose: bool = False,
         print_exceptions=True,
         remote_cache_description: Optional[str] = None,
+        remote_inference_description: Optional[str] = None,
     ) -> Results:
         """
         Runs the Job: conducts Interviews and returns their results.
@@ -431,11 +434,11 @@ class Jobs(Base):
         :param progress_bar: shows a progress bar
         :param stop_on_exception: stops the job if an exception is raised
         :param cache: a cache object to store results
-        :param remote: run the job remotely
         :param check_api_keys: check if the API keys are valid
         :param batch_mode: run the job in batch mode i.e., no expecation of interaction with the user
         :param verbose: prints messages
         :param remote_cache_description: specifies a description for this group of entries in the remote cache
+        :param remote_inference_description: specifies a description for the remote inference job
         """
         from edsl.coop.coop import Coop
 
@@ -446,21 +449,57 @@ class Jobs(Base):
                 "Batch mode is deprecated. Please update your code to not include 'batch_mode' in the 'run' method."
             )
 
-        self.remote = remote
         self.verbose = verbose
 
         try:
             coop = Coop()
-            remote_cache = coop.edsl_settings["remote_caching"]
+            user_edsl_settings = coop.edsl_settings
+            remote_cache = user_edsl_settings["remote_caching"]
+            remote_inference = user_edsl_settings["remote_inference"]
         except Exception:
             remote_cache = False
+            remote_inference = False
+
+        if remote_inference:
+            from edsl.agents.Agent import Agent
+            from edsl.language_models.registry import Model
+            from edsl.results.Result import Result
+            from edsl.results.Results import Results
+            from edsl.scenarios.Scenario import Scenario
+            from edsl.surveys.Survey import Survey
+
+            self._output("Remote inference activated. Sending job to server...")
+            if remote_cache:
+                self._output(
+                    "Remote caching activated. The remote cache will be used for this job."
+                )
 
-        if self.remote:
-            ## TODO: This should be a coop check
-            if os.getenv("EXPECTED_PARROT_API_KEY", None) is None:
-                raise MissingRemoteInferenceError()
-
-        if not self.remote:
+            remote_job_data = coop.remote_inference_create(
+                self,
+                description=remote_inference_description,
+                status="queued",
+            )
+            self._output("Job sent!")
+            # Create mock results object to store job data
+            results = Results(
+                survey=Survey(),
+                data=[
+                    Result(
+                        agent=Agent.example(),
+                        scenario=Scenario.example(),
+                        model=Model(),
+                        iteration=1,
+                        answer={"info": "Remote job details"},
+                    )
+                ],
+            )
+            results.add_columns_from_dict([remote_job_data])
+            if self.verbose:
+                results.select(["info", "uuid", "status", "version"]).print(
+                    format="rich"
+                )
+            return results
+        else:
             if check_api_keys:
                 for model in self.models + [Model()]:
                     if not model.has_valid_api_key():
@@ -471,8 +510,12 @@ class Jobs(Base):
 
         # handle cache
         if cache is None:
+            from edsl.data.CacheHandler import CacheHandler
+
             cache = CacheHandler().get_cache()
         if cache is False:
+            from edsl.data.Cache import Cache
+
             cache = Cache()
 
         if not remote_cache:
@@ -624,6 +667,11 @@ class Jobs(Base):
     @remove_edsl_version
     def from_dict(cls, data: dict) -> Jobs:
         """Creates a Jobs instance from a dictionary."""
+        from edsl import Survey
+        from edsl.agents.Agent import Agent
+        from edsl.language_models.LanguageModel import LanguageModel
+        from edsl.scenarios.Scenario import Scenario
+
         return cls(
             survey=Survey.from_dict(data["survey"]),
             agents=[Agent.from_dict(agent) for agent in data["agents"]],
@@ -639,7 +687,9 @@ class Jobs(Base):
     # Example methods
     #######################
     @classmethod
-    def example(cls, throw_exception_probability=0) -> Jobs:
+    def example(
+        cls, throw_exception_probability: int = 0, randomize: bool = False
+    ) -> Jobs:
         """Return an example Jobs instance.
 
         :param throw_exception_probability: the probability that an exception will be thrown when answering a question. This is useful for testing error handling.
@@ -649,8 +699,12 @@ class Jobs(Base):
 
         """
         import random
+        from uuid import uuid4
         from edsl.questions import QuestionMultipleChoice
-        from edsl import Agent
+        from edsl.agents.Agent import Agent
+        from edsl.scenarios.Scenario import Scenario
+
+        addition = "" if not randomize else str(uuid4())
 
         # (status, question, period)
         agent_answers = {
@@ -689,10 +743,15 @@ class Jobs(Base):
             question_options=["Good", "Great", "OK", "Terrible"],
             question_name="how_feeling_yesterday",
         )
+        from edsl import Survey, ScenarioList
+
         base_survey = Survey(questions=[q1, q2])
 
         scenario_list = ScenarioList(
-            [Scenario({"period": "morning"}), Scenario({"period": "afternoon"})]
+            [
+                Scenario({"period": f"morning{addition}"}),
+                Scenario({"period": "afternoon"}),
+            ]
         )
         job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
 

edsl/jobs/buckets/ModelBuckets.py

@@ -1,4 +1,4 @@
-from edsl.jobs.buckets.TokenBucket import TokenBucket
+# from edsl.jobs.buckets.TokenBucket import TokenBucket
 
 
 class ModelBuckets:
@@ -8,7 +8,7 @@ class ModelBuckets:
     A request is one call to the service. The number of tokens required for a request depends on parameters.
     """
 
-    def __init__(self, requests_bucket: TokenBucket, tokens_bucket: TokenBucket):
+    def __init__(self, requests_bucket: "TokenBucket", tokens_bucket: "TokenBucket"):
         """Initialize the model buckets.
 
         The requests bucket captures requests per unit of time.
@@ -25,9 +25,21 @@ class ModelBuckets:
             tokens_bucket=self.tokens_bucket + other.tokens_bucket,
         )
 
+    def turbo_mode_on(self):
+        """Set the refill rate to infinity for both buckets."""
+        self.requests_bucket.turbo_mode_on()
+        self.tokens_bucket.turbo_mode_on()
+
+    def turbo_mode_off(self):
+        """Restore the refill rate to its original value for both buckets."""
+        self.requests_bucket.turbo_mode_off()
+        self.tokens_bucket.turbo_mode_off()
+
     @classmethod
     def infinity_bucket(cls, model_name: str = "not_specified") -> "ModelBuckets":
         """Create a bucket with infinite capacity and refill rate."""
+        from edsl.jobs.buckets.TokenBucket import TokenBucket
+
         return cls(
             requests_bucket=TokenBucket(
                 bucket_name=model_name,