persidict 0.0.7__py3-none-any.whl → 0.0.8__py3-none-any.whl

persidict/__init__.py

@@ -21,4 +21,5 @@ as a pickle or a json S3 object.
 
 from .safe_str_tuple import SafeStrTuple
 from .persi_dict import PersiDict
-from .file_dir_dict import FileDirDict
+from .file_dir_dict import FileDirDict
+from .s3_dict import S3Dict

persidict/file_dir_dict.py

@@ -40,7 +40,8 @@ class FileDirDict(PersiDict):
     def __init__(self
                  , dir_name: str = "FileDirDict"
                  , file_type: str = "pkl"
-                 , immutable_items:bool = False):
+                 , immutable_items:bool = False
+                 , digest_len:int = 8):
         """A constructor defines location of the store and file format to use.
 
         dir_name is a directory that will contain all the files in
@@ -51,7 +52,8 @@ class FileDirDict(PersiDict):
         to store values.
         """
 
-        super().__init__(immutable_items = immutable_items)
+        super().__init__(immutable_items = immutable_items
+                ,digetst_len = digest_len)
 
         self.file_type = file_type
 

persidict/s3_dict.py

@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import boto3
+
+from .safe_str_tuple import SafeStrTuple
+from .safe_str_tuple_signing import sign_safe_str_tuple, unsign_safe_str_tuple
+from .persi_dict import PersiDict
+from .file_dir_dict import FileDirDict
+
+
+class S3Dict(PersiDict):
+    """ A persistent dictionary that stores key-value pairs as S3 objects.
+
+    A new object is created for each key-value pair.
+
+    A key is either an objectname (a 'filename' without an extension),
+    or a sequence of folder names (object name prefixes) that ends
+    with an objectname. A value can be an instance of any Python type,
+    and will be stored as an S3-object.
+
+    S3Dict can store objects in binary objects (as pickles)
+    or in human-readable texts objects (using jsonpickles).
+
+    Unlike in native Python dictionaries, insertion order is not preserved.
+    """
+
+
+    def __init__(self, bucket_name:str
+                 , region:str = None
+                 , root_prefix:str = ""
+                 , dir_name:str = "S3_Dict"
+                 , file_type:str = "pkl"
+                 , immutable_items:bool = False
+                 , digest_len:int = 8
+                 ,*args ,**kwargs):
+        """A constructor defines location of the store and object format to use.
+
+        bucket_name and region define an S3 location of the storage
+        that will contain all the objects in the S3_Dict.
+        If the bucket does not exist, it will be created.
+
+        root_prefix is a common S3 prefix for all objectnames in a dictionary.
+
+        dir_name is a local directory that will be used to store tmp files.
+
+        file_type can take one of two values: "pkl" or "json".
+        It defines which object format will be used by S3_Dict
+        to store values.
+        """
+
+        super().__init__(immutable_items = immutable_items, digest_len = 0)
+        self.file_type = file_type
+
+        self.local_cache = FileDirDict(
+            dir_name = dir_name
+            , file_type = file_type
+            , immutable_items = immutable_items
+            , digest_len = digest_len)
+
+        self.region = region
+        if region is None:
+            self.s3_client = boto3.client('s3')
+        else:
+            self.s3_client = boto3.client('s3', region_name=region)
+
+        self.bucket = self.s3_client.create_bucket(Bucket=bucket_name)
+        self.bucket_name = bucket_name
+
+        self.root_prefix=root_prefix
+        if len(self.root_prefix) and self.root_prefix[-1] != "/":
+            self.root_prefix += "/"
+
+
+    def __repr__(self):
+        """Return repr(self)."""
+
+        repr_str = super().__repr__()
+        repr_str = repr_str[:-1] + f", dir_name={self.local_cache.base_dir}"
+        repr_str += f", file_type={self.file_type}"
+        repr_str += f", region={self.region}"
+        repr_str += f", bucket_name={self.bucket_name}"
+        repr_str += f", root_prefix={self.root_prefix}"
+        repr_str += " )"
+
+        return repr_str
+
+
+    def _build_full_objectname(self, key:SafeStrTuple) -> str:
+        """ Convert SafeStrTuple into an S3 objectname. """
+
+        key = sign_safe_str_tuple(key, self.digest_len)
+        objectname = self.root_prefix +  "/".join(key)+ "." + self.file_type
+        return objectname
+
+
+    def __contains__(self, key:SafeStrTuple) -> bool:
+        """True if the dictionary has the specified key, else False. """
+        if self.immutable_items:
+            file_name = self.local_cache._build_full_path(
+                key, create_subdirs=True)
+            if os.path.exists(file_name):
+                return True
+        try:
+            obj_name = self._build_full_objectname(key)
+            self.s3_client.head_object(Bucket=self.bucket_name, Key=obj_name)
+            return True
+        except:
+            return False
+
+
+    def __getitem__(self, key:SafeStrTuple) -> Any:
+        """X.__getitem__(y) is an equivalent to X[y]. """
+
+        file_name = self.local_cache._build_full_path(key, create_subdirs=True)
+
+        if self.immutable_items:
+            try:
+                result = self.local_cache._read_from_file(file_name)
+                return result
+            except:
+                pass
+
+        obj_name = self._build_full_objectname(key)
+        self.s3_client.download_file(self.bucket_name, obj_name, file_name)
+        result = self.local_cache._read_from_file(file_name)
+        if not self.immutable_items:
+            os.remove(file_name)
+
+        return result
+
+
+    def __setitem__(self, key:SafeStrTuple, value:Any):
+        """Set self[key] to value. """
+
+        file_name = self.local_cache._build_full_path(key, create_subdirs=True)
+        obj_name = self._build_full_objectname(key)
+
+        if self.immutable_items:
+            key_is_present = False
+            if os.path.exists(file_name):
+                key_is_present = True
+            else:
+                try:
+                    self.s3_client.head_object(
+                        Bucket=self.bucket_name, Key=obj_name)
+                    key_is_present = True
+                except:
+                    key_is_present = False
+
+            assert not key_is_present, "Can't modify an immutable item"
+
+        self.local_cache._save_to_file(file_name, value)
+        self.s3_client.upload_file(file_name, self.bucket_name, obj_name)
+        if not self.immutable_items:
+            os.remove(file_name)
+
+
+    def __delitem__(self, key:SafeStrTuple):
+        """Delete self[key]. """
+
+        assert not self.immutable_items, "Can't delete an immutable item"
+        obj_name = self._build_full_objectname(key)
+        self.s3_client.delete_object(Bucket = self.bucket_name, Key = obj_name)
+        file_name = self.local_cache._build_full_path(key)
+        if os.path.isfile(file_name):
+            os.remove(file_name)
+
+
+    def __len__(self) -> int:
+        """Return len(self). """
+
+        num_files = 0
+        suffix = "." + self.file_type
+
+        paginator = self.s3_client.get_paginator("list_objects")
+        page_iterator = paginator.paginate(
+            Bucket=self.bucket_name, Prefix = self.root_prefix)
+
+        for page in page_iterator:
+            if "Contents" in page:
+                for key in page["Contents"]:
+                    obj_name = key["Key"]
+                    if obj_name.endswith(suffix):
+                        num_files += 1
+
+        return num_files
+
+
+    def _generic_iter(self, iter_type: str):
+        """Underlying implementation for .items()/.keys()/.values() iterators"""
+        assert iter_type in {"keys", "values", "items"}
+        suffix = "." + self.file_type
+        ext_len = len(self.file_type) + 1
+        prefix_len = len(self.root_prefix)
+
+        def splitter(full_name: str) -> SafeStrTuple:
+            assert full_name.startswith(self.root_prefix)
+            result = full_name[prefix_len:-ext_len].split(sep="/")
+            return SafeStrTuple(result)
+
+        def step():
+            paginator = self.s3_client.get_paginator("list_objects")
+            page_iterator = paginator.paginate(
+                Bucket=self.bucket_name, Prefix = self.root_prefix)
+
+            for page in page_iterator:
+                if "Contents" in page:
+                    for key in page["Contents"]:
+                        obj_name = key["Key"]
+                        if not obj_name.endswith(suffix):
+                            continue
+                        obj_key = splitter(obj_name)
+                        if iter_type == "keys":
+                            yield unsign_safe_str_tuple(
+                                obj_key, self.digest_len)
+                        elif iter_type == "values":
+                            yield self[obj_key]
+                        else:
+                            yield (unsign_safe_str_tuple(
+                                obj_key, self.digest_len), self[obj_key])
+
+        return step()
+
+
+    def get_subdict(self, key:SafeStrTuple) -> S3Dict:
+        """Get a subdictionary containing items with the same prefix_key.
+
+        This method is absent in the original dict API.
+        """
+        if len(key):
+            key = SafeStrTuple(key)
+            key = sign_safe_str_tuple(key, self.digest_len)
+            full_root_prefix = self.root_prefix +  "/".join(key)
+        else:
+            full_root_prefix = self.root_prefix
+
+        new_dir_path = self.local_cache._build_full_path(
+            key, create_subdirs = True, is_file_path = False)
+
+        return S3Dict(
+            bucket_name = self.bucket_name
+            , region = self.region
+            , root_prefix = full_root_prefix
+            , dir_name = new_dir_path
+            , file_type = self.file_type
+            , immutable_items = self.immutable_items)
+
+
+    def mtimestamp(self,key:SafeStrTuple) -> float:
+        """Get last modification time (in seconds, Unix epoch time).
+
+        This method is absent in the original dict API.
+        """
+        #TODO: check work with timezones
+        obj_name = self._build_full_objectname(key)
+        response = self.s3_client.head_object(Bucket=self.bucket_name, Key=obj_name)
+        return response["LastModified"].timestamp()

{persidict-0.0.7.dist-info → persidict-0.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: persidict
-Version: 0.0.7
+Version: 0.0.8
 Summary: Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud.
 Home-page: https://github.com/vladlpavlov/persidict
 Author: Vlad (Volodymyr) Pavlov

persidict-0.0.8.dist-info/RECORD

@@ -0,0 +1,11 @@
+persidict/__init__.py,sha256=v28AnmkmoLSqZmKAoSsN3a3B86N1ohUcDih5te8vCiA,882
+persidict/file_dir_dict.py,sha256=8tS5nKUHgFITcxW5NwAnZdtINV87ap4PqXN19_uRP_M,9002
+persidict/persi_dict.py,sha256=g9BiEjnfoExXMxJHm3kONt_8iPkhCYqy_imUKDtzimg,7254
+persidict/s3_dict.py,sha256=AXYY3gHeKRxoO4B51Evd1GV1I_t9HmsPp9q1QLJLwoo,9151
+persidict/safe_str_tuple.py,sha256=jnXq8Har5nJyKtu0Qy2tOiRZ9DuIrY1YNpkwMHc3f1I,3416
+persidict/safe_str_tuple_signing.py,sha256=OvGyaNOhx_I518nTifz3sRbej-vxtuRWqd4w1aEp8FY,3743
+persidict-0.0.8.dist-info/LICENSE,sha256=oWpRWnm32aM5jVoboapPqgHi-drh1OPeZrrafyE1zTs,1113
+persidict-0.0.8.dist-info/METADATA,sha256=ipg31oYZJI_cTvUo9taVRyk-az_fzH1dFJJX-JKBqvo,2114
+persidict-0.0.8.dist-info/WHEEL,sha256=5sUXSg9e4bi7lTLOHcm6QEYwO5TIF1TNbTSVFVjcJcc,92
+persidict-0.0.8.dist-info/top_level.txt,sha256=7Kr8wnF-PGd513PbAORWUpp7Bi09VIZVTQS7ZN-dJXc,10
+persidict-0.0.8.dist-info/RECORD,,

persidict-0.0.7.dist-info/RECORD

@@ -1,10 +0,0 @@
-persidict/__init__.py,sha256=MSoe2iiRMVJJ_OETL0eC812ZXu9cY4djLm3f5JVmiV0,854
-persidict/file_dir_dict.py,sha256=4FF1lFx1QLnDPy_nz-6rQyf4SSR2ErUILXGelvEHlGY,8922
-persidict/persi_dict.py,sha256=g9BiEjnfoExXMxJHm3kONt_8iPkhCYqy_imUKDtzimg,7254
-persidict/safe_str_tuple.py,sha256=jnXq8Har5nJyKtu0Qy2tOiRZ9DuIrY1YNpkwMHc3f1I,3416
-persidict/safe_str_tuple_signing.py,sha256=OvGyaNOhx_I518nTifz3sRbej-vxtuRWqd4w1aEp8FY,3743
-persidict-0.0.7.dist-info/LICENSE,sha256=oWpRWnm32aM5jVoboapPqgHi-drh1OPeZrrafyE1zTs,1113
-persidict-0.0.7.dist-info/METADATA,sha256=YKuhGwoKzQnkxBnp3f8depNfh8Qa95a-toblFbdrVXs,2114
-persidict-0.0.7.dist-info/WHEEL,sha256=5sUXSg9e4bi7lTLOHcm6QEYwO5TIF1TNbTSVFVjcJcc,92
-persidict-0.0.7.dist-info/top_level.txt,sha256=7Kr8wnF-PGd513PbAORWUpp7Bi09VIZVTQS7ZN-dJXc,10
-persidict-0.0.7.dist-info/RECORD,,