edsl 0.1.30.dev1__tar.gz → 0.1.30.dev3__tar.gz

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edsl
-Version: 0.1.30.dev1
+Version: 0.1.30.dev3
 Summary: Create and analyze LLM-based surveys
 Home-page: https://www.expectedparrot.com/
 License: MIT

edsl-0.1.30.dev3/edsl/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.30.dev3"

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/agents/Agent.py

@@ -4,7 +4,8 @@ from __future__ import annotations
 import copy
 import inspect
 import types
-from typing import Any, Callable, Optional, Union, Dict, Sequence
+from typing import Callable, Optional, Union
+from uuid import uuid4
 from edsl.Base import Base
 
 from edsl.exceptions.agents import (
@@ -688,13 +689,14 @@ class Agent(Base):
         return table
 
     @classmethod
-    def example(cls) -> Agent:
-        """Return an example agent.
+    def example(cls, randomize: bool = False) -> Agent:
+        """
+        Returns an example Agent instance.
 
-        >>> Agent.example()
-        Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5})
+        :param randomize: If True, adds a random string to the value of an example key.
         """
-        return cls(traits={"age": 22, "hair": "brown", "height": 5.5})
+        addition = "" if not randomize else str(uuid4())
+        return cls(traits={"age": 22, "hair": f"brown{addition}", "height": 5.5})
 
     def code(self) -> str:
         """Return the code for the agent.

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/agents/AgentList.py

@@ -11,23 +11,15 @@ Example usage:
 """
 
 from __future__ import annotations
+import csv
+import json
 from collections import UserList
-from typing import Optional, Union, Sequence, List, Any
+from typing import Any, List, Optional, Union
 from rich import print_json
 from rich.table import Table
-import json
-import csv
-
-
 from simpleeval import EvalWithCompoundTypes
-
 from edsl.Base import Base
-
-# from edsl.agents import Agent
-from edsl.utilities.decorators import (
-    add_edsl_version,
-    remove_edsl_version,
-)
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
 
 class AgentList(UserList, Base):
@@ -239,17 +231,15 @@ class AgentList(UserList, Base):
         return cls(agents)
 
     @classmethod
-    def example(cls) -> "AgentList":
-        """Return an example AgentList.
-
-        >>> al = AgentList.example()
-        >>> len(al)
-        2
+    def example(cls, randomize: bool = False) -> AgentList:
+        """
+        Returns an example AgentList instance.
 
+        :param randomize: If True, uses Agent's randomize method.
         """
         from edsl.agents.Agent import Agent
 
-        return cls([Agent.example(), Agent.example()])
+        return cls([Agent.example(randomize), Agent.example(randomize)])
 
     @classmethod
     def from_list(self, trait_name: str, values: List[Any]):

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/agents/Invigilator.py

@@ -74,15 +74,14 @@ class InvigilatorAI(PromptConstructorMixin, InvigilatorBase):
 
         This cleans up the raw response to make it suitable to pass to AgentResponseDict.
         """
-        # not actually used, but this removes the temptation to delete agent from the signature
         _ = agent
         try:
             response = question._validate_answer(raw_response)
         except Exception as e:
+            """If the response is invalid, remove it from the cache and raise the exception."""
             self._remove_from_cache(raw_response)
             raise e
 
-        # breakpoint()
         question_dict = self.survey.question_names_to_questions()
         for other_question, answer in self.current_answers.items():
             if other_question in question_dict:
@@ -95,12 +94,10 @@ class InvigilatorAI(PromptConstructorMixin, InvigilatorBase):
                     question_dict[new_question].comment = answer
 
         combined_dict = {**question_dict, **scenario}
-        # print("combined_dict: ", combined_dict)
-        # print("response: ", response)
-        # breakpoint()
         answer = question._translate_answer_code_to_answer(
             response["answer"], combined_dict
         )
+        # breakpoint()
         data = {
             "answer": answer,
             "comment": response.get(
@@ -111,6 +108,8 @@ class InvigilatorAI(PromptConstructorMixin, InvigilatorBase):
             "cached_response": raw_response.get("cached_response", None),
             "usage": raw_response.get("usage", {}),
             "raw_model_response": raw_model_response,
+            "cache_used": raw_response.get("cache_used", False),
+            "cache_key": raw_response.get("cache_key", None),
         }
         return AgentResponseDict(**data)
 

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/data/Cache.py

@@ -7,17 +7,10 @@ import json
 import os
 import warnings
 from typing import Optional, Union
-import time
-from edsl.config import CONFIG
-from edsl.data.CacheEntry import CacheEntry
-
-# from edsl.data.SQLiteDict import SQLiteDict
 from edsl.Base import Base
+from edsl.data.CacheEntry import CacheEntry
 from edsl.utilities.utilities import dict_hash
-from edsl.utilities.decorators import (
-    add_edsl_version,
-    remove_edsl_version,
-)
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
 
 class Cache(Base):
@@ -41,6 +34,7 @@ class Cache(Base):
         data: Optional[Union["SQLiteDict", dict]] = None,
         immediate_write: bool = True,
         method=None,
+        verbose=False,
     ):
         """
         Create two dictionaries to store the cache data.
@@ -59,6 +53,7 @@ class Cache(Base):
         self.new_entries = {}
         self.new_entries_to_write_later = {}
         self.coop = None
+        self.verbose = verbose
 
         self.filename = filename
         if filename and data:
@@ -122,7 +117,7 @@ class Cache(Base):
         system_prompt: str,
         user_prompt: str,
         iteration: int,
-    ) -> Union[None, str]:
+    ) -> tuple(Union[None, str], str):
         """
         Fetch a value (LLM output) from the cache.
 
@@ -135,7 +130,7 @@ class Cache(Base):
         Return None if the response is not found.
 
         >>> c = Cache()
-        >>> c.fetch(model="gpt-3", parameters="default", system_prompt="Hello", user_prompt="Hi", iteration=1) is None
+        >>> c.fetch(model="gpt-3", parameters="default", system_prompt="Hello", user_prompt="Hi", iteration=1)[0] is None
         True
 
 
@@ -151,8 +146,13 @@ class Cache(Base):
         )
         entry = self.data.get(key, None)
         if entry is not None:
+            if self.verbose:
+                print(f"Cache hit for key: {key}")
             self.fetched_data[key] = entry
-        return None if entry is None else entry.output
+        else:
+            if self.verbose:
+                print(f"Cache miss for key: {key}")
+        return None if entry is None else entry.output, key
 
     def store(
         self,
@@ -354,6 +354,9 @@ class Cache(Base):
         for key, entry in self.new_entries_to_write_later.items():
             self.data[key] = entry
 
+        if self.filename:
+            self.write(self.filename)
+
     ####################
     # DUNDER / USEFUL
     ####################
@@ -470,12 +473,18 @@ class Cache(Base):
         webbrowser.open("file://" + filepath)
 
     @classmethod
-    def example(cls) -> Cache:
+    def example(cls, randomize: bool = False) -> Cache:
         """
-        Return an example Cache.
-        The example Cache has one entry.
+        Returns an example Cache instance.
+
+        :param randomize: If True, uses CacheEntry's randomize method.
         """
-        return cls(data={CacheEntry.example().key: CacheEntry.example()})
+        return cls(
+            data={
+                CacheEntry.example(randomize).key: CacheEntry.example(),
+                CacheEntry.example(randomize).key: CacheEntry.example(),
+            }
+        )
 
 
 if __name__ == "__main__":

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/data/CacheEntry.py

@@ -2,11 +2,8 @@ from __future__ import annotations
 import json
 import datetime
 import hashlib
-import random
 from typing import Optional
-
-
-# TODO: Timestamp should probably be float?
+from uuid import uuid4
 
 
 class CacheEntry:
@@ -151,10 +148,12 @@ class CacheEntry:
     @classmethod
     def example(cls, randomize: bool = False) -> CacheEntry:
         """
-        Returns a CacheEntry example.
+        Returns an example CacheEntry instance.
+
+        :param randomize: If True, adds a random string to the system prompt.
         """
-        # if random, create a random number for 0-100
-        addition = "" if not randomize else str(random.randint(0, 1000))
+        # if random, create a uuid
+        addition = "" if not randomize else str(uuid4())
         return CacheEntry(
             model="gpt-3.5-turbo",
             parameters={"temperature": 0.5},

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/data_transfer_models.py

@@ -17,6 +17,8 @@ class AgentResponseDict(UserDict):
         cached_response=None,
         raw_model_response=None,
         simple_model_raw_response=None,
+        cache_used=None,
+        cache_key=None,
     ):
         """Initialize the AgentResponseDict object."""
         usage = usage or {"prompt_tokens": 0, "completion_tokens": 0}
@@ -30,5 +32,7 @@ class AgentResponseDict(UserDict):
                 "cached_response": cached_response,
                 "raw_model_response": raw_model_response,
                 "simple_model_raw_response": simple_model_raw_response,
+                "cache_used": cache_used,
+                "cache_key": cache_key,
             }
         )

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/jobs/Jobs.py

@@ -461,6 +461,13 @@ class Jobs(Base):
             remote_inference = False
 
         if remote_inference:
+            from edsl.agents.Agent import Agent
+            from edsl.language_models.registry import Model
+            from edsl.results.Result import Result
+            from edsl.results.Results import Results
+            from edsl.scenarios.Scenario import Scenario
+            from edsl.surveys.Survey import Survey
+
             self._output("Remote inference activated. Sending job to server...")
             if remote_cache:
                 self._output(
@@ -680,7 +687,9 @@ class Jobs(Base):
     # Example methods
     #######################
     @classmethod
-    def example(cls, throw_exception_probability=0) -> Jobs:
+    def example(
+        cls, throw_exception_probability: int = 0, randomize: bool = False
+    ) -> Jobs:
         """Return an example Jobs instance.
 
         :param throw_exception_probability: the probability that an exception will be thrown when answering a question. This is useful for testing error handling.
@@ -690,10 +699,13 @@ class Jobs(Base):
 
         """
         import random
+        from uuid import uuid4
         from edsl.questions import QuestionMultipleChoice
         from edsl.agents.Agent import Agent
         from edsl.scenarios.Scenario import Scenario
 
+        addition = "" if not randomize else str(uuid4())
+
         # (status, question, period)
         agent_answers = {
             ("Joyful", "how_feeling", "morning"): "OK",
@@ -736,7 +748,10 @@ class Jobs(Base):
         base_survey = Survey(questions=[q1, q2])
 
         scenario_list = ScenarioList(
-            [Scenario({"period": "morning"}), Scenario({"period": "afternoon"})]
+            [
+                Scenario({"period": f"morning{addition}"}),
+                Scenario({"period": "afternoon"}),
+            ]
         )
         job = base_survey.by(scenario_list).by(joy_agent, sad_agent)
 

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/jobs/buckets/ModelBuckets.py

@@ -25,6 +25,16 @@ class ModelBuckets:
             tokens_bucket=self.tokens_bucket + other.tokens_bucket,
         )
 
+    def turbo_mode_on(self):
+        """Set the refill rate to infinity for both buckets."""
+        self.requests_bucket.turbo_mode_on()
+        self.tokens_bucket.turbo_mode_on()
+
+    def turbo_mode_off(self):
+        """Restore the refill rate to its original value for both buckets."""
+        self.requests_bucket.turbo_mode_off()
+        self.tokens_bucket.turbo_mode_off()
+
     @classmethod
     def infinity_bucket(cls, model_name: str = "not_specified") -> "ModelBuckets":
         """Create a bucket with infinite capacity and refill rate."""

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/jobs/buckets/TokenBucket.py

@@ -17,11 +17,28 @@ class TokenBucket:
         self.bucket_name = bucket_name
         self.bucket_type = bucket_type
         self.capacity = capacity  # Maximum number of tokens
+        self._old_capacity = capacity
         self.tokens = capacity  # Current number of available tokens
         self.refill_rate = refill_rate  # Rate at which tokens are refilled
+        self._old_refill_rate = refill_rate
         self.last_refill = time.monotonic()  # Last refill time
-
         self.log: List[Any] = []
+        self.turbo_mode = False
+
+    def turbo_mode_on(self):
+        """Set the refill rate to infinity."""
+        if self.turbo_mode:
+            pass
+        else:
+            self.turbo_mode = True
+            self.capacity = float("inf")
+            self.refill_rate = float("inf")
+
+    def turbo_mode_off(self):
+        """Restore the refill rate to its original value."""
+        self.turbo_mode = False
+        self.capacity = self._old_capacity
+        self.refill_rate = self._old_refill_rate
 
     def __add__(self, other) -> "TokenBucket":
         """Combine two token buckets.
@@ -98,7 +115,7 @@ class TokenBucket:
             raise ValueError(msg)
         while self.tokens < amount:
             self.refill()
-            await asyncio.sleep(0.1)  # Sleep briefly to prevent busy waiting
+            await asyncio.sleep(0.01)  # Sleep briefly to prevent busy waiting
         self.tokens -= amount
 
         now = time.monotonic()

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/jobs/interviews/Interview.py

@@ -30,14 +30,14 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
 
     def __init__(
         self,
-        agent: Agent,
-        survey: Survey,
-        scenario: Scenario,
-        model: Type[LanguageModel],
-        debug: bool = False,
+        agent: "Agent",
+        survey: "Survey",
+        scenario: "Scenario",
+        model: Type["LanguageModel"],
+        debug: Optional[bool] = False,
         iteration: int = 0,
         cache: "Cache" = None,
-        sidecar_model: LanguageModel = None,
+        sidecar_model: "LanguageModel" = None,
     ):
         """Initialize the Interview instance.
 
@@ -99,6 +99,7 @@ class Interview(InterviewStatusMixin, InterviewTaskBuildingMixin):
         if model_buckets is None or hasattr(self.agent, "answer_question_directly"):
             model_buckets = ModelBuckets.infinity_bucket()
 
+        # FOR TESTING
         # model_buckets = ModelBuckets.infinity_bucket()
 
         ## build the tasks using the InterviewTaskBuildingMixin

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -88,6 +88,8 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
                     self.total_interviews.append(interview)
 
     async def run_async(self, cache=None) -> Results:
+        from edsl.results.Results import Results
+
         if cache is None:
             self.cache = Cache()
         else:
@@ -98,6 +100,8 @@ class JobsRunnerAsyncio(JobsRunnerStatusMixin):
         return Results(survey=self.jobs.survey, data=data)
 
     def simple_run(self):
+        from edsl.results.Results import Results
+
         data = asyncio.run(self.run_async())
         return Results(survey=self.jobs.survey, data=data)
 

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/jobs/tasks/QuestionTaskCreator.py

@@ -144,12 +144,20 @@ class QuestionTaskCreator(UserList):
             self.task_status = TaskStatus.FAILED
             raise e
 
-        if "cached_response" in results:
-            if results["cached_response"]:
-                # Gives back the tokens b/c the API was not called.
-                self.tokens_bucket.add_tokens(requested_tokens)
-                self.requests_bucket.add_tokens(1)
-                self.from_cache = True
+        ## This isn't working
+        # breakpoint()
+        if results.get("cache_used", False):
+            self.tokens_bucket.add_tokens(requested_tokens)
+            self.requests_bucket.add_tokens(1)
+            self.from_cache = True
+            # print("Turning on turbo!")
+            self.tokens_bucket.turbo_mode_on()
+            self.requests_bucket.turbo_mode_on()
+        else:
+            # breakpoint()
+            # print("Turning off turbo!")
+            self.tokens_bucket.turbo_mode_off()
+            self.requests_bucket.turbo_mode_off()
 
         _ = results.pop("cached_response", None)
 

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/language_models/LanguageModel.py

@@ -323,12 +323,10 @@ class LanguageModel(
             image_hash = hashlib.md5(encoded_image.encode()).hexdigest()
             cache_call_params["user_prompt"] = f"{user_prompt} {image_hash}"
 
-        cached_response = cache.fetch(**cache_call_params)
-
+        cached_response, cache_key = cache.fetch(**cache_call_params)
         if cached_response:
             response = json.loads(cached_response)
             cache_used = True
-            cache_key = None
         else:
             remote_call = hasattr(self, "remote") and self.remote
             f = (
@@ -340,7 +338,7 @@ class LanguageModel(
             if encoded_image:
                 params["encoded_image"] = encoded_image
             response = await f(**params)
-            cache_key = cache.store(
+            new_cache_key = cache.store(
                 user_prompt=user_prompt,
                 model=str(self.model),
                 parameters=self.parameters,
@@ -348,6 +346,7 @@ class LanguageModel(
                 response=response,
                 iteration=iteration,
             )
+            assert new_cache_key == cache_key
             cache_used = False
 
         return response, cache_used, cache_key
@@ -412,7 +411,7 @@ class LanguageModel(
 
         dict_response.update(
             {
-                "cached_used": cache_used,
+                "cache_used": cache_used,
                 "cache_key": cache_key,
                 "usage": raw_response.get("usage", {}),
                 "raw_model_response": raw_response,

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/notebooks/Notebook.py

@@ -1,14 +1,11 @@
 """A Notebook is a utility class that allows you to easily share/pull ipynbs from Coop."""
 
+from __future__ import annotations
 import json
 from typing import Dict, List, Optional
-
-
+from uuid import uuid4
 from edsl.Base import Base
-from edsl.utilities.decorators import (
-    add_edsl_version,
-    remove_edsl_version,
-)
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 
 
 class Notebook(Base):
@@ -192,10 +189,13 @@ class Notebook(Base):
         return table
 
     @classmethod
-    def example(cls) -> "Notebook":
+    def example(cls, randomize: bool = False) -> Notebook:
         """
-        Return an example Notebook.
+        Returns an example Notebook instance.
+
+        :param randomize: If True, adds a random string one of the cells' output.
         """
+        addition = "" if not randomize else str(uuid4())
         cells = [
             {
                 "cell_type": "markdown",
@@ -210,7 +210,7 @@ class Notebook(Base):
                     {
                         "name": "stdout",
                         "output_type": "stream",
-                        "text": "Hello world!\n",
+                        "text": f"Hello world!\n{addition}",
                     }
                 ],
                 "source": 'print("Hello world!")',

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/questions/QuestionFreeText.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 import textwrap
 from typing import Any, Optional
+from uuid import uuid4
 from edsl.questions.QuestionBase import QuestionBase
 
 
@@ -65,9 +66,10 @@ class QuestionFreeText(QuestionBase):
         return question_html_content
 
     @classmethod
-    def example(cls) -> QuestionFreeText:
+    def example(cls, randomize: bool = False) -> QuestionFreeText:
         """Return an example instance of a free text question."""
-        return cls(question_name="how_are_you", question_text="How are you?")
+        addition = "" if not randomize else str(uuid4())
+        return cls(question_name="how_are_you", question_text=f"How are you?{addition}")
 
 
 def main():

{edsl-0.1.30.dev1 → edsl-0.1.30.dev3}/edsl/results/Results.py

@@ -949,7 +949,7 @@ class Results(UserList, Mixins, Base):
         return Results(survey=self.survey, data=new_data, created_columns=None)
 
     @classmethod
-    def example(cls, debug: bool = False) -> Results:
+    def example(cls, debug: bool = False, randomize: bool = False) -> Results:
         """Return an example `Results` object.
 
         Example usage:
@@ -962,7 +962,7 @@ class Results(UserList, Mixins, Base):
         from edsl.data.Cache import Cache
 
         c = Cache()
-        job = Jobs.example()
+        job = Jobs.example(randomize=randomize)
         results = job.run(cache=c, debug=debug)
         return results