edsl 0.1.39__py3-none-any.whl → 0.1.39.dev1__py3-none-any.whl

edsl/data/Cache.py

@@ -6,10 +6,12 @@ from __future__ import annotations
 import json
 import os
 import warnings
-from typing import Optional, Union, TYPE_CHECKING
+import copy
+from typing import Optional, Union
 from edsl.Base import Base
-
-from edsl.utilities.remove_edsl_version import remove_edsl_version
+from edsl.data.CacheEntry import CacheEntry
+from edsl.utilities.utilities import dict_hash
+from edsl.utilities.decorators import remove_edsl_version
 from edsl.exceptions.cache import CacheError
 
 
@@ -81,6 +83,10 @@ class Cache(Base):
 
         self._perform_checks()
 
+    def rich_print(sefl):
+        pass
+        # raise NotImplementedError("This method is not implemented yet.")
+
     def code(sefl):
         pass
         # raise NotImplementedError("This method is not implemented yet.")
@@ -195,7 +201,6 @@ class Cache(Base):
         >>> len(c)
         1
         """
-        from edsl.data.CacheEntry import CacheEntry
 
         entry = CacheEntry(
             model=model,
@@ -221,7 +226,6 @@ class Cache(Base):
 
         :param write_now: Whether to write to the cache immediately (similar to `immediate_write`).
         """
-        from edsl.data.CacheEntry import CacheEntry
 
         for key, value in new_data.items():
             if key in self.data:
@@ -242,8 +246,6 @@ class Cache(Base):
 
         :param write_now: Whether to write to the cache immediately (similar to `immediate_write`).
         """
-        from edsl.data.CacheEntry import CacheEntry
-
         with open(filename, "a+") as f:
             f.seek(0)
             lines = f.readlines()
@@ -287,8 +289,8 @@ class Cache(Base):
 
         CACHE_PATH = CONFIG.get("EDSL_DATABASE_PATH")
         path = CACHE_PATH.replace("sqlite:///", "")
-        # db_path = os.path.join(os.path.dirname(path), "data.db")
-        return cls.from_sqlite_db(path)
+        db_path = os.path.join(os.path.dirname(path), "data.db")
+        return cls.from_sqlite_db(db_path=db_path)
 
     @classmethod
     def from_jsonl(cls, jsonlfile: str, db_path: Optional[str] = None) -> Cache:
@@ -351,8 +353,7 @@ class Cache(Base):
                 f.write(json.dumps({key: value.to_dict()}) + "\n")
 
     def to_scenario_list(self):
-        from edsl.scenarios.ScenarioList import ScenarioList
-        from edsl.scenarios.Scenario import Scenario
+        from edsl import ScenarioList, Scenario
 
         scenarios = []
         for key, value in self.data.items():
@@ -362,32 +363,12 @@ class Cache(Base):
             scenarios.append(s)
         return ScenarioList(scenarios)
 
-    def __floordiv__(self, other: "Cache") -> "Cache":
-        """
-        Return a new Cache containing entries that are in self but not in other.
-        Uses // operator as alternative to subtraction.
-
-        :param other: Another Cache object to compare against
-        :return: A new Cache object containing unique entries
-
-        >>> from edsl.data.CacheEntry import CacheEntry
-        >>> ce1 = CacheEntry.example(randomize = True)
-        >>> ce2 = CacheEntry.example(randomize = True)
-        >>> ce2 = CacheEntry.example(randomize = True)
-        >>> c1 = Cache(data={ce1.key: ce1, ce2.key: ce2})
-        >>> c2 = Cache(data={ce1.key: ce1})
-        >>> c3 = c1 // c2
-        >>> len(c3)
-        1
-        >>> c3.data[ce2.key] == ce2
-        True
-        """
-        if not isinstance(other, Cache):
-            raise CacheError("Can only compare two caches")
-
-        diff_data = {k: v for k, v in self.data.items() if k not in other.data}
-        return Cache(data=diff_data, immediate_write=self.immediate_write)
-
+    ####################
+    # REMOTE
+    ####################
+    # TODO: Make this work
+    # - Need to decide whether the cache belongs to a user and what can be shared
+    # - I.e., some cache entries? all or nothing?
     @classmethod
     def from_url(cls, db_path=None) -> Cache:
         """
@@ -413,10 +394,11 @@ class Cache(Base):
         if self.filename:
             self.write(self.filename)
 
+    ####################
+    # DUNDER / USEFUL
+    ####################
     def __hash__(self):
         """Return the hash of the Cache."""
-        from edsl.utilities.utilities import dict_hash
-
         return dict_hash(self.to_dict(add_edsl_version=False))
 
     def to_dict(self, add_edsl_version=True) -> dict:
@@ -432,6 +414,12 @@ class Cache(Base):
     def _summary(self):
         return {"EDSL Class": "Cache", "Number of entries": len(self.data)}
 
+    def _repr_html_(self):
+        # from edsl.utilities.utilities import data_to_html
+        # return data_to_html(self.to_dict())
+        footer = f"<a href={self.__documentation__}>(docs)</a>"
+        return str(self.summary(format="html")) + footer
+
     def table(
         self,
         *fields,
@@ -455,8 +443,6 @@ class Cache(Base):
     @remove_edsl_version
     def from_dict(cls, data) -> Cache:
         """Construct a Cache from a dictionary."""
-        from edsl.data.CacheEntry import CacheEntry
-
         newdata = {k: CacheEntry.from_dict(v) for k, v in data.items()}
         return cls(data=newdata)
 
@@ -499,8 +485,6 @@ class Cache(Base):
         """
         Create an example input for a 'fetch' operation.
         """
-        from edsl.data.CacheEntry import CacheEntry
-
         return CacheEntry.fetch_input_example()
 
     def to_html(self):
@@ -557,8 +541,6 @@ class Cache(Base):
 
         :param randomize: If True, uses CacheEntry's randomize method.
         """
-        from edsl.data.CacheEntry import CacheEntry
-
         return cls(
             data={
                 CacheEntry.example(randomize).key: CacheEntry.example(),

edsl/data/CacheEntry.py

@@ -5,12 +5,8 @@ import hashlib
 from typing import Optional
 from uuid import uuid4
 
-from edsl.utilities.decorators import remove_edsl_version
 
-from edsl.Base import RepresentationMixin
-
-
-class CacheEntry(RepresentationMixin):
+class CacheEntry:
     """
     A Class to represent a cache entry.
     """
@@ -82,11 +78,11 @@ class CacheEntry(RepresentationMixin):
         d = {k: value for k, value in self.__dict__.items() if k in self.key_fields}
         return self.gen_key(**d)
 
-    def to_dict(self, add_edsl_version=True) -> dict:
+    def to_dict(self) -> dict:
         """
         Returns a dictionary representation of a CacheEntry.
         """
-        d = {
+        return {
             "model": self.model,
             "parameters": self.parameters,
             "system_prompt": self.system_prompt,
@@ -95,12 +91,19 @@ class CacheEntry(RepresentationMixin):
             "iteration": self.iteration,
             "timestamp": self.timestamp,
         }
-        # if add_edsl_version:
-        #     from edsl import __version__
 
-        #     d["edsl_version"] = __version__
-        #     d["edsl_class_name"] = self.__class__.__name__
-        return d
+    def _repr_html_(self) -> str:
+        """
+        Returns an HTML representation of a CacheEntry.
+        """
+        # from edsl.utilities.utilities import data_to_html
+        # return data_to_html(self.to_dict())
+        d = self.to_dict()
+        data = [[k, v] for k, v in d.items()]
+        from tabulate import tabulate
+
+        table = str(tabulate(data, headers=["keys", "values"], tablefmt="html"))
+        return f"<pre>{table}</pre>"
 
     def keys(self):
         return list(self.to_dict().keys())

edsl/data/CacheHandler.py

@@ -3,19 +3,19 @@ import ast
 import json
 import os
 import shutil
-from typing import TYPE_CHECKING
+import sqlite3
+from edsl.config import CONFIG
+from edsl.data.Cache import Cache
+from edsl.data.CacheEntry import CacheEntry
+from edsl.data.SQLiteDict import SQLiteDict
 
-if TYPE_CHECKING:
-    from edsl.data.Cache import Cache
-    from edsl.data.CacheEntry import CacheEntry
+from edsl.config import CONFIG
 
 
-def set_session_cache(cache: "Cache") -> None:
+def set_session_cache(cache: Cache) -> None:
     """
     Set the session cache.
     """
-    from edsl.config import CONFIG
-
     CONFIG.EDSL_SESSION_CACHE = cache
 
 
@@ -23,8 +23,6 @@ def unset_session_cache() -> None:
     """
     Unset the session cache.
     """
-    from edsl.config import CONFIG
-
     if hasattr(CONFIG, "EDSL_SESSION_CACHE"):
         del CONFIG.EDSL_SESSION_CACHE
 
@@ -34,11 +32,7 @@ class CacheHandler:
     This CacheHandler figures out what caches are available and does migrations, as needed.
     """
 
-    @property
-    def CACHE_PATH(self):
-        from edsl.config import CONFIG
-
-        return CONFIG.get("EDSL_DATABASE_PATH")
+    CACHE_PATH = CONFIG.get("EDSL_DATABASE_PATH")
 
     def __init__(self, test: bool = False):
         self.test = test
@@ -58,22 +52,16 @@ class CacheHandler:
             if notify:
                 print(f"Created cache directory: {dir_path}")
 
-    def gen_cache(self) -> "Cache":
+    def gen_cache(self) -> Cache:
         """
         Generate a Cache object.
         """
-        from edsl.data.Cache import Cache
-
         if self.test:
             return Cache(data={})
 
-        from edsl.config import CONFIG
-
         if hasattr(CONFIG, "EDSL_SESSION_CACHE"):
             return CONFIG.EDSL_SESSION_CACHE
 
-        from edsl.data.SQLiteDict import SQLiteDict
-
         cache = Cache(data=SQLiteDict(self.CACHE_PATH))
         return cache
 
@@ -88,8 +76,6 @@ class CacheHandler:
         if not os.path.exists(os.path.join(os.getcwd(), path)):
             return old_data
         try:
-            import sqlite3
-
             conn = sqlite3.connect(path)
             with conn:
                 cur = conn.cursor()
@@ -122,8 +108,6 @@ class CacheHandler:
         entry_dict["user_prompt"] = entry_dict.pop("prompt")
         parameters = entry_dict["parameters"]
         entry_dict["parameters"] = ast.literal_eval(parameters)
-        from edsl.data.CacheEntry import CacheEntry
-
         entry = CacheEntry(**entry_dict)
         return entry
 
@@ -133,7 +117,7 @@ class CacheHandler:
     ###############
     # NOT IN USE
     ###############
-    def from_sqlite(uri="new_edsl_cache.db") -> dict[str, "CacheEntry"]:
+    def from_sqlite(uri="new_edsl_cache.db") -> dict[str, CacheEntry]:
         """
         Read in a new-style sqlite cache and return a dictionary of dictionaries.
         """
@@ -147,7 +131,7 @@ class CacheHandler:
                 newdata[entry.key] = entry
         return newdata
 
-    def from_jsonl(filename="edsl_cache.jsonl") -> dict[str, "CacheEntry"]:
+    def from_jsonl(filename="edsl_cache.jsonl") -> dict[str, CacheEntry]:
         """Read in a jsonl file and return a dictionary of CacheEntry objects."""
         with open(filename, "a+") as f:
             f.seek(0)
@@ -162,7 +146,4 @@ class CacheHandler:
 
 
 if __name__ == "__main__":
-    # ch = CacheHandler()
-    import doctest
-
-    doctest.testmod()
+    ch = CacheHandler()

edsl/data/RemoteCacheSync.py

@@ -1,166 +1,71 @@
-from typing import List, Dict, Any, Optional, TYPE_CHECKING, Callable
-from dataclasses import dataclass
-from contextlib import AbstractContextManager
-from collections import UserList
-
-if TYPE_CHECKING:
-    from .Cache import Cache
-    from edsl.coop.coop import Coop
-    from .CacheEntry import CacheEntry
-
-from logging import Logger
-
-
-class CacheKeyList(UserList):
-    def __init__(self, data: List[str]):
-        super().__init__(data)
-        self.data = data
-
-    def __repr__(self):
-        import reprlib
-
-        keys_repr = reprlib.repr(self.data)
-        return f"CacheKeyList({keys_repr})"
-
-
-class CacheEntriesList(UserList):
-    def __init__(self, data: List["CacheEntry"]):
-        super().__init__(data)
-        self.data = data
-
-    def __repr__(self):
-        import reprlib
-
-        entries_repr = reprlib.repr(self.data)
-        return f"CacheEntries({entries_repr})"
-
-    def to_cache(self) -> "Cache":
-        from edsl.data.Cache import Cache
-
-        return Cache({entry.key: entry for entry in self.data})
-
-
-@dataclass
-class CacheDifference:
-    client_missing_entries: CacheEntriesList
-    server_missing_keys: List[str]
-
-    def __repr__(self):
-        """Returns a string representation of the CacheDifference object."""
-        import reprlib
-
-        missing_entries_repr = reprlib.repr(self.client_missing_entries)
-        missing_keys_repr = reprlib.repr(self.server_missing_keys)
-        return f"CacheDifference(client_missing_entries={missing_entries_repr}, server_missing_keys={missing_keys_repr})"
-
-
-class RemoteCacheSync(AbstractContextManager):
-    """Synchronizes a local cache with a remote cache.
-
-    Handles bidirectional synchronization:
-    - Downloads missing entries from remote to local cache
-    - Uploads new local entries to remote cache
-    """
-
+class RemoteCacheSync:
     def __init__(
-        self,
-        coop: "Coop",
-        cache: "Cache",
-        output_func: Callable,
-        remote_cache: bool = True,
-        remote_cache_description: str = "",
+        self, coop, cache, output_func, remote_cache=True, remote_cache_description=""
     ):
-        """
-        Initializes a RemoteCacheSync object.
-
-        :param coop: Coop object for interacting with the remote cache
-        :param cache: Cache object for local cache
-        :param output_func: Function for outputting messages
-        :param remote_cache: Whether to enable remote cache synchronization
-        :param remote_cache_description: Description for remote cache entries
-
-        """
         self.coop = coop
         self.cache = cache
         self._output = output_func
-        self.remote_cache_enabled = remote_cache
+        self.remote_cache = remote_cache
+        self.old_entry_keys = []
+        self.new_cache_entries = []
         self.remote_cache_description = remote_cache_description
-        self.initial_cache_keys = []
 
-    def __enter__(self) -> "RemoteCacheSync":
-        if self.remote_cache_enabled:
+    def __enter__(self):
+        if self.remote_cache:
             self._sync_from_remote()
-            self.initial_cache_keys = list(self.cache.keys())
+            self.old_entry_keys = list(self.cache.keys())
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
-        if self.remote_cache_enabled:
+        if self.remote_cache:
             self._sync_to_remote()
         return False  # Propagate exceptions
 
-    def _get_cache_difference(self) -> CacheDifference:
-        """Retrieves differences between local and remote caches."""
-        diff = self.coop.remote_cache_get_diff(self.cache.keys())
-        return CacheDifference(
-            client_missing_entries=diff.get("client_missing_cacheentries", []),
-            server_missing_keys=diff.get("server_missing_cacheentry_keys", []),
+    def _sync_from_remote(self):
+        cache_difference = self.coop.remote_cache_get_diff(self.cache.keys())
+        client_missing_cacheentries = cache_difference.get(
+            "client_missing_cacheentries", []
         )
+        missing_entry_count = len(client_missing_cacheentries)
 
-    def _sync_from_remote(self) -> None:
-        """Downloads missing entries from remote cache to local cache."""
-        diff: CacheDifference = self._get_cache_difference()
-        missing_count = len(diff.client_missing_entries)
-
-        if missing_count == 0:
+        if missing_entry_count > 0:
+            self._output(
+                f"Updating local cache with {missing_entry_count:,} new "
+                f"{'entry' if missing_entry_count == 1 else 'entries'} from remote..."
+            )
+            self.cache.add_from_dict(
+                {entry.key: entry for entry in client_missing_cacheentries}
+            )
+            self._output("Local cache updated!")
+        else:
             self._output("No new entries to add to local cache.")
-            return
 
-        self._output(
-            f"Updating local cache with {missing_count:,} new "
-            f"{'entry' if missing_count == 1 else 'entries'} from remote..."
+    def _sync_to_remote(self):
+        cache_difference = self.coop.remote_cache_get_diff(self.cache.keys())
+        server_missing_cacheentry_keys = cache_difference.get(
+            "server_missing_cacheentry_keys", []
         )
-
-        self.cache.add_from_dict(
-            {entry.key: entry for entry in diff.client_missing_entries}
-        )
-        self._output("Local cache updated!")
-
-    def _get_entries_to_upload(self, diff: CacheDifference) -> CacheEntriesList:
-        """Determines which entries need to be uploaded to remote cache."""
-        # Get entries for keys missing from server
-        server_missing_entries = CacheEntriesList(
-            [
-                entry
-                for key in diff.server_missing_keys
-                if (entry := self.cache.data.get(key)) is not None
-            ]
-        )
-
-        # Get newly added entries since sync started
-        new_entries = CacheEntriesList(
-            [
-                entry
-                for entry in self.cache.values()
-                if entry.key not in self.initial_cache_keys
-            ]
-        )
-
-        return server_missing_entries + new_entries
-
-    def _sync_to_remote(self) -> None:
-        """Uploads new local entries to remote cache."""
-        diff: CacheDifference = self._get_cache_difference()
-        entries_to_upload: CacheEntriesList = self._get_entries_to_upload(diff)
-        upload_count = len(entries_to_upload)
-
-        if upload_count > 0:
+        server_missing_cacheentries = [
+            entry
+            for key in server_missing_cacheentry_keys
+            if (entry := self.cache.data.get(key)) is not None
+        ]
+
+        new_cache_entries = [
+            entry
+            for entry in self.cache.values()
+            if entry.key not in self.old_entry_keys
+        ]
+        server_missing_cacheentries.extend(new_cache_entries)
+        new_entry_count = len(server_missing_cacheentries)
+
+        if new_entry_count > 0:
             self._output(
-                f"Updating remote cache with {upload_count:,} new "
-                f"{'entry' if upload_count == 1 else 'entries'}..."
+                f"Updating remote cache with {new_entry_count:,} new "
+                f"{'entry' if new_entry_count == 1 else 'entries'}..."
             )
-
             self.coop.remote_cache_create_many(
-                entries_to_upload,
+                server_missing_cacheentries,
                 visibility="private",
                 description=self.remote_cache_description,
             )
@@ -171,16 +76,3 @@ class RemoteCacheSync(AbstractContextManager):
         self._output(
             f"There are {len(self.cache.keys()):,} entries in the local cache."
         )
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod()
-
-    from edsl.coop.coop import Coop
-    from edsl.data.Cache import Cache
-    from edsl.data.CacheEntry import CacheEntry
-
-    r = RemoteCacheSync(Coop(), Cache(), print)
-    diff = r._get_cache_difference()

edsl/data/__init__.py

@@ -1,5 +1,4 @@
-# from edsl.data.CacheEntry import CacheEntry
-# from edsl.data.SQLiteDict import SQLiteDict
+from edsl.data.CacheEntry import CacheEntry
+from edsl.data.SQLiteDict import SQLiteDict
 from edsl.data.Cache import Cache
-
-# from edsl.data.CacheHandler import CacheHandler
+from edsl.data.CacheHandler import CacheHandler

edsl/data_transfer_models.py

@@ -1,5 +1,6 @@
 from typing import NamedTuple, Dict, List, Optional, Any
 from dataclasses import dataclass, fields
+import reprlib
 
 
 class ModelInputs(NamedTuple):
@@ -55,8 +56,6 @@ class ImageInfo:
     encoded_image: str
 
     def __repr__(self):
-        import reprlib
-
         reprlib_instance = reprlib.Repr()
         reprlib_instance.maxstring = 30  # Limit the string length for the encoded image
 

edsl/enums.py

@@ -1,7 +1,6 @@
 """Enums for the different types of questions, language models, and inference services."""
 
 from enum import Enum
-from typing import Literal
 
 
 class EnumWithChecks(Enum):
@@ -68,32 +67,6 @@ class InferenceServiceType(EnumWithChecks):
     PERPLEXITY = "perplexity"
 
 
-# unavoidable violation of the DRY principle but it is necessary
-# checked w/ a unit test to make sure consistent with services in enums.py
-InferenceServiceLiteral = Literal[
-    "bedrock",
-    "deep_infra",
-    "replicate",
-    "openai",
-    "google",
-    "test",
-    "anthropic",
-    "groq",
-    "azure",
-    "ollama",
-    "mistral",
-    "together",
-    "perplexity",
-]
-
-available_models_urls = {
-    "anthropic": "https://docs.anthropic.com/en/docs/about-claude/models",
-    "openai": "https://platform.openai.com/docs/models/gp",
-    "groq": "https://console.groq.com/docs/models",
-    "google": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models",
-}
-
-
 service_to_api_keyname = {
     InferenceServiceType.BEDROCK.value: "TBD",
     InferenceServiceType.DEEP_INFRA.value: "DEEP_INFRA_API_KEY",