tilebox-workflows 0.47.0__py3-none-any.whl → 0.48.0__py3-none-any.whl

tilebox/workflows/cache.py

@@ -1,13 +1,16 @@
+import contextlib
 import warnings
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
 from io import BytesIO
 from pathlib import Path
+from pathlib import PurePosixPath as ObjectPath
 
 import boto3
 from botocore.exceptions import ClientError
 from google.cloud.exceptions import NotFound
 from google.cloud.storage import Blob, Bucket
+from obstore.store import ObjectStore
 
 
 class JobCache(ABC):
@@ -62,6 +65,53 @@ class NoCache(JobCache):
         return self
 
 
+class ObstoreCache(JobCache):
+    def __init__(self, store: ObjectStore, prefix: str | ObjectPath = ObjectPath(".")) -> None:
+        """A cache implementation backed by an obstore ObjectStore.
+
+        This cache implementation is the recommended way of working with the cache, as it provides a unified interface
+        for working with different object stores, while also providing a way to transparently work with local files
+        as well.
+
+        Args:
+            store: The object store to use for the cache.
+            prefix: A path prefix to append to all objects stored in the cache. Defaults to no prefix.
+        """
+        self.store = store
+        self.prefix = ObjectPath(prefix)
+
+    def __contains__(self, key: str) -> bool:
+        with contextlib.suppress(OSError):
+            self.store.get(str(self.prefix / key))
+            return True  # if get is successful, we know the key is in the cache
+
+        return False
+
+    def __setitem__(self, key: str, value: bytes) -> None:
+        self.store.put(str(self.prefix / key), value)
+
+    def __delitem__(self, key: str) -> None:
+        try:
+            self.store.delete(str(self.prefix / key))
+        except OSError:
+            raise KeyError(f"{key} is not cached!") from None
+
+    def __getitem__(self, key: str) -> bytes:
+        try:
+            entry = self.store.get(str(self.prefix / key))
+            return bytes(entry.bytes())
+        except OSError:
+            raise KeyError(f"{key} is not cached!") from None
+
+    def __iter__(self) -> Iterator[str]:
+        for obj in self.store.list_with_delimiter(str(self.prefix))["objects"]:
+            path: str = obj["path"]
+            yield path.removeprefix(str(self.prefix) + "/")
+
+    def group(self, key: str) -> "ObstoreCache":
+        return ObstoreCache(self.store, prefix=str(self.prefix / key))
+
+
 class InMemoryCache(JobCache):
     def __init__(self) -> None:
         """A simple in-memory cache implementation.
@@ -153,7 +203,7 @@ class LocalFileSystemCache(JobCache):
         Args:
             root: File system path where the cache will be stored. Defaults to "cache" in the current working directory.
         """
-        self.root = root if isinstance(root, Path) else Path(root)
+        self.root = Path(root)
 
     def __contains__(self, key: str) -> bool:
         return (self.root / key).exists()
@@ -184,7 +234,7 @@ class LocalFileSystemCache(JobCache):
 
 
 class GoogleStorageCache(JobCache):
-    def __init__(self, bucket: Bucket, prefix: str = "jobs") -> None:
+    def __init__(self, bucket: Bucket, prefix: str | ObjectPath = "jobs") -> None:
         """A cache implementation that stores data in Google Cloud Storage.
 
         Args:
@@ -192,7 +242,9 @@ class GoogleStorageCache(JobCache):
             prefix: A path prefix to append to all objects stored in the cache. Defaults to "jobs".
         """
         self.bucket = bucket
-        self.prefix = Path(prefix)  # we still use pathlib here, because it's easier to work with when joining paths
+        self.prefix = ObjectPath(
+            prefix
+        )  # we still use pathlib here, because it's easier to work with when joining paths
 
     def _blob(self, key: str) -> Blob:
         return self.bucket.blob(str(self.prefix / key))
@@ -228,14 +280,14 @@ class GoogleStorageCache(JobCache):
 
         # make the names relative to the cache prefix (but including the key in the name)
         for blob in blobs:
-            yield str(Path(blob.name).relative_to(self.prefix))
+            yield str(ObjectPath(blob.name).relative_to(self.prefix))
 
     def group(self, key: str) -> "GoogleStorageCache":
         return GoogleStorageCache(self.bucket, prefix=str(self.prefix / key))
 
 
 class AmazonS3Cache(JobCache):
-    def __init__(self, bucket: str, prefix: str = "jobs") -> None:
+    def __init__(self, bucket: str, prefix: str | ObjectPath = "jobs") -> None:
         """A cache implementation that stores data in Amazon S3.
 
         Args:
@@ -243,7 +295,7 @@ class AmazonS3Cache(JobCache):
             prefix: A path prefix to append to all objects stored in the cache. Defaults to "jobs".
         """
         self.bucket = bucket
-        self.prefix = Path(prefix)
+        self.prefix = ObjectPath(prefix)
         with warnings.catch_warnings():
             # https://github.com/boto/boto3/issues/3889
             warnings.filterwarnings("ignore", category=DeprecationWarning, message=".*datetime.utcnow.*")

{tilebox_workflows-0.47.0.dist-info → tilebox_workflows-0.48.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tilebox-workflows
-Version: 0.47.0
+Version: 0.48.0
 Summary: Workflow client and task runner for Tilebox
 Project-URL: Homepage, https://tilebox.com
 Project-URL: Documentation, https://docs.tilebox.com/workflows/introduction

{tilebox_workflows-0.47.0.dist-info → tilebox_workflows-0.48.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 tilebox/workflows/__init__.py,sha256=D6NXvTUjWv0YWN5tYD09p1cFkGs8nGrZ9V7agtjvp8c,636
-tilebox/workflows/cache.py,sha256=GK0lOIuAEhHya4jCe5PNc1T72xuCrs62AhD6zIoNPQk,11253
+tilebox/workflows/cache.py,sha256=ZgVUm2O4Qr1nPXvUVx82YGtiDg4IjUbHpnm5KTfrKvo,13233
 tilebox/workflows/client.py,sha256=L8MZXZ-yDutu91sD1am24yfANLCaasRXO8ITJHg5UgE,5680
 tilebox/workflows/data.py,sha256=8DBWZJonsLKCMMF02jpD0xfEENbrSUqANFvBYzMb1eY,29559
 tilebox/workflows/interceptors.py,sha256=yfo6pCxUdhb0EC1J506k1ge4S_BAl83TAFYxCcxu8sU,1799
@@ -42,6 +42,6 @@ tilebox/workflows/workflows/v1/task_pb2_grpc.py,sha256=nkQjtsDiql1ofbSxhDXbPkySd
 tilebox/workflows/workflows/v1/workflows_pb2.py,sha256=rGwIydUP4osLD_fG3QmHaqs42mKmRoCqtwihJTLJ314,3990
 tilebox/workflows/workflows/v1/workflows_pb2.pyi,sha256=qiDQUM2Vlu_izQvaSDlK5GqKYGsNJgtm0bo9zW-qNjU,1633
 tilebox/workflows/workflows/v1/workflows_pb2_grpc.py,sha256=36Vp_TIxtS-MRBZHECf84fHFbnrm3-UizCsMOlz7qfo,8529
-tilebox_workflows-0.47.0.dist-info/METADATA,sha256=SccnKvYADXLBSRIokW3HK4nbqdz4l7IbRkAhXlziZns,3948
-tilebox_workflows-0.47.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-tilebox_workflows-0.47.0.dist-info/RECORD,,
+tilebox_workflows-0.48.0.dist-info/METADATA,sha256=g_7EhTNi4DXYUYheJjhks2sGQU4hwdFcUCFhhwxu6fU,3948
+tilebox_workflows-0.48.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+tilebox_workflows-0.48.0.dist-info/RECORD,,