persidict 0.32.8__py3-none-any.whl → 0.34.2__py3-none-any.whl

persidict/file_dir_dict.py

@@ -11,6 +11,7 @@ from __future__ import annotations
 
 import os
 import random
+import tempfile
 import time
 from typing import Any, Optional
 
@@ -79,6 +80,10 @@ class FileDirDict(PersiDict):
 
         assert file_type == replace_unsafe_chars(file_type, "")
         self.file_type = file_type
+        if self.file_type == "__etag__":
+            raise ValueError(
+                "file_type cannot be 'etag' as it is a reserved"
+                " extension for S3 caching.")
 
         if (base_class_for_values is None or
                 not issubclass(base_class_for_values,str)):
@@ -90,13 +95,7 @@ class FileDirDict(PersiDict):
         if os.path.isfile(base_dir):
             raise ValueError(f"{base_dir} is a file, not a directory.")
 
-        try: # extra protection to better handle concurrent access
-            if not os.path.isdir(base_dir):
-                os.mkdir(base_dir)
-        except:
-            time.sleep(random.random()/random.randint(1, 3))
-            if not os.path.isdir(base_dir):
-                os.mkdir(base_dir)
+        os.makedirs(base_dir, exist_ok=True)
         assert os.path.isdir(base_dir)
 
         # self.base_dir_param = _base_dir
@@ -137,7 +136,12 @@ class FileDirDict(PersiDict):
 
 
     def __len__(self) -> int:
-        """ Get the number of key-value pairs in the dictionary."""
+        """ Get the number of key-value pairs in the dictionary.
+
+        WARNING: This operation can be slow on large dictionaries as it
+        needs to recursively walk the entire base directory.
+        Avoid using it in performance-sensitive code.
+        """
 
         suffix = "." + self.file_type
         return sum(1 for _, _, files in os.walk(self._base_dir)
@@ -150,6 +154,9 @@ class FileDirDict(PersiDict):
         if self.immutable_items:
             raise KeyError("Can't clear a dict that contains immutable items")
 
+        # we can't use shutil.rmtree() because
+        # there may be overlapping dictionaries
+        # with different file_type-s
         for subdir_info in os.walk(self._base_dir, topdown=False):
             (subdir_name, _, files) = subdir_info
             suffix = "." + self.file_type
@@ -172,17 +179,8 @@ class FileDirDict(PersiDict):
         dir_names = key[:-1] if is_file_path else key
 
         if create_subdirs:
-            current_dir = dir_names[0]
-            for dir_name in dir_names[1:]:
-                new_dir = os.path.join(current_dir, dir_name)
-                try: # extra protection to better handle concurrent access
-                    if not os.path.isdir(new_dir):
-                        os.mkdir(new_dir)
-                except:
-                    time.sleep(random.random()/random.randint(1, 3))
-                    if not os.path.isdir(new_dir):
-                        os.mkdir(new_dir)
-                current_dir = new_dir
+            dir_path = os.path.join(*dir_names)
+            os.makedirs(dir_path, exist_ok=True)
 
         if is_file_path:
             file_name = key[-1] + "." + self.file_type
@@ -269,25 +267,50 @@ class FileDirDict(PersiDict):
         for i in range(n_retries):
             try:
                 return self._read_from_file_impl(file_name)
-            except:
-                time.sleep(random.random()/random.randint(1, 10))
-
-        return self._read_from_file_impl(file_name)
+            except Exception as e:
+                if i < n_retries - 1:
+                    time.sleep(random.uniform(0.01, 0.1) * (2 ** i))
+                else:
+                    raise e
 
 
     def _save_to_file_impl(self, file_name:str, value:Any) -> None:
         """Save a value to a file. """
 
-        if self.file_type == "pkl":
-            with open(file_name, 'wb') as f:
-                joblib.dump(value, f, compress='lz4')
-        elif self.file_type == "json":
-            with open(file_name, 'w') as f:
-                f.write(jsonpickle.dumps(value, indent=4))
-        else:
-            with open(file_name, 'w') as f:
-                f.write(value)
+        dir_name = os.path.dirname(file_name)
+        # Use a temporary file and atomic rename to prevent data corruption
+        fd, temp_path = tempfile.mkstemp(dir=dir_name, prefix=".__tmp__")
+
+        try:
+            if self.file_type == "pkl":
+                with open(fd, 'wb') as f:
+                    joblib.dump(value, f, compress='lz4')
+                    f.flush()
+                    os.fsync(f.fileno())
+            elif self.file_type == "json":
+                with open(fd, 'w') as f:
+                    f.write(jsonpickle.dumps(value, indent=4))
+                    f.flush()
+                    os.fsync(f.fileno())
+            else:
+                with open(fd, 'w') as f:
+                    f.write(value)
+                    f.flush()
+                    os.fsync(f.fileno())
+            os.replace(temp_path, file_name)
+            try:
+                if os.name == 'posix':
+                    dir_fd = os.open(dir_name, os.O_RDONLY)
+                    try:
+                        os.fsync(dir_fd)
+                    finally:
+                        os.close(dir_fd)
+            except OSError:
+                pass
 
+        except:
+            os.remove(temp_path)
+            raise
 
     def _save_to_file(self, file_name:str, value:Any) -> None:
         """Save a value to a file. """
@@ -297,16 +320,17 @@ class FileDirDict(PersiDict):
             raise ValueError("When base_class_for_values is not str,"
                 + " file_type must be pkl or json.")
 
-        n_retries = 3
+        n_retries = 8
         # extra protections to better handle concurrent writes
         for i in range(n_retries):
-            try: # extra protections to better handle concurrent writes
+            try:
                 self._save_to_file_impl(file_name, value)
                 return
-            except:
-                time.sleep(random.random()/random.randint(1, 5))
-
-        self._save_to_file_impl(file_name, value)
+            except Exception as e:
+                if i < n_retries - 1:
+                    time.sleep(random.uniform(0.01, 0.1) * (2 ** i))
+                else:
+                    raise e
 
 
     def __contains__(self, key:PersiDictKey) -> bool:
@@ -381,16 +405,9 @@ class FileDirDict(PersiDict):
 
         def splitter(dir_path: str):
             """Transform a dirname into a PersiDictKey key"""
-            splitted_str = []
             if dir_path == ".":
-                return splitted_str
-            while True:
-                head, tail = os.path.split(dir_path)
-                splitted_str = [tail] + splitted_str
-                dir_path = head
-                if len(head) == 0:
-                    break
-            return tuple(splitted_str)
+                return []
+            return dir_path.split(os.sep)
 
         def step():
             suffix = "." + self.file_type
@@ -439,7 +456,6 @@ class FileDirDict(PersiDict):
 
     def random_key(self) -> PersiDictKey | None:
         # canonicalise extension once
-        early_exit_cap = 10_000
         ext = None
         if self.file_type:
             ext = self.file_type.lower()
@@ -467,9 +483,6 @@ class FileDirDict(PersiDict):
                             seen += 1
                             if random.random() < 1 / seen:  # reservoir k=1
                                 winner = ent.path
-                            # early‑exit when cap reached
-                            if early_exit_cap and seen >= early_exit_cap:
-                                return self._build_key_from_full_path(os.path.abspath(winner))
             except PermissionError:
                 continue
 

persidict/s3_dict.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
 import os
+import tempfile
 from typing import Any, Optional
 
 import boto3
+from botocore.exceptions import ClientError
+
 import parameterizable
 from parameterizable.dict_sorter import sort_dict_by_keys
 
@@ -60,16 +63,19 @@ class S3Dict(PersiDict):
         check types of values in the dictionary. If not specified,
         no type checking will be performed and all types will be allowed.
 
-        file_type is extension, which will be used for all files in the dictionary.
+        file_type is an extension, which will be used for all files in the dictionary.
         If file_type has one of two values: "lz4" or "json", it defines
         which file format will be used by FileDirDict to store values.
         For all other values of file_type, the file format will always be plain
-        text. "lz4" or "json" allow to store arbitrary Python objects,
+        text. "lz4" or "json" allow storing arbitrary Python objects,
         while all other file_type-s only work with str objects.
         """
 
         super().__init__(immutable_items = immutable_items, digest_len = 0)
         self.file_type = file_type
+        if self.file_type == "__etag__":
+            raise ValueError(
+                "file_type cannot be 'etag' as it is a reserved extension for caching.")
 
         self.local_cache = FileDirDict(
             base_dir= base_dir
@@ -152,26 +158,102 @@ class S3Dict(PersiDict):
             return False
 
 
+    def _write_etag_file(self, file_name: str, etag: str):
+        """Atomically write the ETag to its cache file."""
+        if not etag:
+            return
+        etag_file_name = file_name + ".__etag__"
+        dir_name = os.path.dirname(etag_file_name)
+        # Write to a temporary file and then rename for atomicity
+        fd, temp_path = tempfile.mkstemp(dir=dir_name)
+        try:
+            with os.fdopen(fd, "w") as f:
+                f.write(etag)
+                f.flush()
+                os.fsync(f.fileno())
+            os.replace(temp_path, etag_file_name)
+            try:
+                if os.name == 'posix':
+                    dir_fd = os.open(dir_name, os.O_RDONLY)
+                    try:
+                        os.fsync(dir_fd)
+                    finally:
+                        os.close(dir_fd)
+            except OSError:
+                pass
+        except:
+            os.remove(temp_path)
+            raise
+
+
     def __getitem__(self, key:PersiDictKey) -> Any:
         """X.__getitem__(y) is an equivalent to X[y]. """
 
         key = SafeStrTuple(key)
         file_name = self.local_cache._build_full_path(key, create_subdirs=True)
 
-        if self.immutable_items:
+        if self.immutable_items and os.path.exists(file_name):
+            return self.local_cache._read_from_file(file_name)
+
+        obj_name = self._build_full_objectname(key)
+
+        cached_etag = None
+        etag_file_name = file_name + ".__etag__"
+        if not self.immutable_items and os.path.exists(file_name) and os.path.exists(
+                etag_file_name):
+            with open(etag_file_name, "r") as f:
+                cached_etag = f.read()
+
+        try:
+            get_kwargs = {'Bucket': self.bucket_name, 'Key': obj_name}
+            if cached_etag:
+                get_kwargs['IfNoneMatch'] = cached_etag
+
+            response = self.s3_client.get_object(**get_kwargs)
+
+            # 200 OK: object was downloaded, either because it's new or changed.
+            s3_etag = response.get("ETag")
+            body = response['Body']
+
+            dir_name = os.path.dirname(file_name)
+            fd, temp_path = tempfile.mkstemp(dir=dir_name, prefix=".__tmp__")
+
             try:
-                result = self.local_cache._read_from_file(file_name)
-                return result
+                with os.fdopen(fd, 'wb') as f:
+                    # Stream body to file to avoid loading all in memory
+                    for chunk in body.iter_chunks():
+                        f.write(chunk)
+                    f.flush()
+                    os.fsync(f.fileno())
+                os.replace(temp_path, file_name)
+                try:
+                    if os.name == 'posix':
+                        dir_fd = os.open(dir_name, os.O_RDONLY)
+                        try:
+                            os.fsync(dir_fd)
+                        finally:
+                            os.close(dir_fd)
+                except OSError:
+                    pass
             except:
-                pass
+                os.remove(temp_path)  # Clean up temp file on failure
+                raise
 
-        obj_name = self._build_full_objectname(key)
-        self.s3_client.download_file(self.bucket_name, obj_name, file_name)
-        result = self.local_cache._read_from_file(file_name)
-        if not self.immutable_items:
-            os.remove(file_name)
+            self._write_etag_file(file_name, s3_etag)
+
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code")
+            if e.response['ResponseMetadata']['HTTPStatusCode'] == 304:
+                # 304 Not Modified: our cached version is up-to-date.
+                # The file will be read from cache at the end of the function.
+                pass
+            elif e.response.get("Error", {}).get("Code") == 'NoSuchKey':
+                raise KeyError(f"Key {key} not found in S3 bucket {self.bucket_name}")
+            else:
+                # Re-raise other client errors (e.g., permissions, throttling)
+                raise
 
-        return result
+        return self.local_cache._read_from_file(file_name)
 
 
     def __setitem__(self, key:PersiDictKey, value:Any):
@@ -196,28 +278,27 @@ class S3Dict(PersiDict):
                     + f"but it is {type(value)} instead." )
 
         key = SafeStrTuple(key)
-        file_name = self.local_cache._build_full_path(key, create_subdirs=True)
-        obj_name = self._build_full_objectname(key)
 
-        if self.immutable_items:
-            key_is_present = False
-            if os.path.exists(file_name):
-                key_is_present = True
-            else:
-                try:
-                    self.s3_client.head_object(
-                        Bucket=self.bucket_name, Key=obj_name)
-                    key_is_present = True
-                except:
-                    key_is_present = False
+        if self.immutable_items and key in self:
+            raise KeyError("Can't modify an immutable item")
 
-            if key_is_present:
-                raise KeyError("Can't modify an immutable item")
+        file_name = self.local_cache._build_full_path(key, create_subdirs=True)
+        obj_name = self._build_full_objectname(key)
 
         self.local_cache._save_to_file(file_name, value)
         self.s3_client.upload_file(file_name, self.bucket_name, obj_name)
-        if not self.immutable_items:
-            os.remove(file_name)
+
+        try:
+            head = self.s3_client.head_object(
+                Bucket=self.bucket_name, Key=obj_name)
+            s3_etag = head.get("ETag")
+            self._write_etag_file(file_name, s3_etag)
+        except ClientError:
+            # If we can't get ETag, we should remove any existing etag file
+            # to force a re-download on the next __getitem__ call.
+            etag_file_name = file_name + ".__etag__"
+            if os.path.exists(etag_file_name):
+                os.remove(etag_file_name)
 
 
     def __delitem__(self, key:PersiDictKey):
@@ -231,10 +312,17 @@ class S3Dict(PersiDict):
         file_name = self.local_cache._build_full_path(key)
         if os.path.isfile(file_name):
             os.remove(file_name)
-
+        etag_file_name = file_name + ".__etag__"
+        if os.path.isfile(etag_file_name):
+            os.remove(etag_file_name)
 
     def __len__(self) -> int:
-        """Return len(self). """
+        """Return len(self).
+
+        WARNING: This operation can be very slow and costly on large S3 buckets
+        as it needs to iterate over all objects in the dictionary's prefix.
+        Avoid using it in performance-sensitive code.
+        """
 
         num_files = 0
         suffix = "." + self.file_type

persidict/safe_str_tuple.py

@@ -43,7 +43,7 @@ class SafeStrTuple(Sequence, Hashable):
             elif isinstance(a, str):
                 assert len(a) > 0
                 assert len(a) < SAFE_STRING_MAX_LENGTH
-                assert len(set(a) - SAFE_CHARS_SET) == 0
+                assert all(c in SAFE_CHARS_SET for c in a)
                 candidate_strings.append(a)
             elif _is_sequence_not_mapping(a):
                 if len(a) > 0:

persidict/write_once_dict.py

@@ -116,17 +116,19 @@ class WriteOnceDict(PersiDict):
         """
         check_needed = False
 
-        try: # extra protections to better handle concurrent writes
-            if key in self._wrapped_dict:
-                check_needed = True
-            else:
-                self._wrapped_dict[key] = value
-        except:
-            time.sleep(random.random()/random.randint(1,5))
-            if key in self._wrapped_dict:
-                check_needed = True
-            else:
-                self._wrapped_dict[key] = value
+        n_retries = 8
+        for i in range(n_retries):
+            try:  # extra protections to better handle concurrent writes
+                if key in self._wrapped_dict:
+                    check_needed = True
+                else:
+                    self._wrapped_dict[key] = value
+                break
+            except Exception as e:
+                if i < n_retries - 1:
+                    time.sleep(random.uniform(0.01, 0.1) * (2 ** i))
+                else:
+                    raise e
 
         if not key in self._wrapped_dict:
             raise KeyError(

{persidict-0.32.8.dist-info → persidict-0.34.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: persidict
-Version: 0.32.8
+Version: 0.34.2
 Summary: Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud.
 Keywords: persistence,dicts,distributed,parallel
 Author: Vlad (Volodymyr) Pavlov
@@ -21,7 +21,6 @@ Requires-Dist: joblib
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: jsonpickle
-Requires-Dist: joblib
 Requires-Dist: deepdiff
 Requires-Dist: boto3 ; extra == 'aws'
 Requires-Dist: boto3 ; extra == 'dev'

{persidict-0.32.8.dist-info → persidict-0.34.2.dist-info}/RECORD

@@ -1,14 +1,14 @@
 persidict/.DS_Store,sha256=1lFlJ5EFymdzGAUAaI30vcaaLHt3F1LwpG7xILf9jsM,6148
 persidict/__init__.py,sha256=CDOSJGgCnyRTkGUTzaeg3Cqsxwx0-0EFieOtldXwAls,1380
-persidict/file_dir_dict.py,sha256=SijMlIbte9kNbEyryjK4-cXakQl35gCAo1HaO1WcaU4,17589
+persidict/file_dir_dict.py,sha256=Dr4gdIC5uqykRgNca1pI_M_jEd_9FGjys0BvzAJR0JU,17804
 persidict/jokers.py,sha256=kX4bE-jKWTM2ki7JOmm_2uJS8zm8u6InZ_V12xo2ImI,1436
 persidict/overlapping_multi_dict.py,sha256=a-lUbmY15_HrDq6jSIt8F8tJboqbeYiuRQeW4elf_oU,2663
 persidict/persi_dict.py,sha256=SF6aWs6kCeeW-bZ9HJwx0sPX7Xav_aURqeSZ-j5quv0,14266
-persidict/s3_dict.py,sha256=awEIu7ehh4FgKircC10lcHp86xvUctXA_6jxUmC3Hy8,12480
+persidict/s3_dict.py,sha256=j_Fb73wbkOrsneu3a48VUpDDXqVSTtEa0sRI-c_Kbjk,16104
 persidict/safe_chars.py,sha256=HjK1MwROYy_U9ui-rhg1i3nGkj52K4OFWD-wCCcnJ7Y,536
-persidict/safe_str_tuple.py,sha256=cTk5BL3r-yE62EKf7VngTaUpZAdsATJPIjiCGqQzkyU,3717
+persidict/safe_str_tuple.py,sha256=xyIzxlCKmvnNHkFFKWtcBREefxZ0-HLxoH_epYDt8qg,3719
 persidict/safe_str_tuple_signing.py,sha256=5uCjAVZRqOou-KpDZw-Exboc3-3vuayJMqrrt8aZ0ck,3742
-persidict/write_once_dict.py,sha256=NBzaw38zxWVbvCj8OR4T-7w6K41qNrr0gpyr23CRcNQ,6424
-persidict-0.32.8.dist-info/WHEEL,sha256=Jb20R3Ili4n9P1fcwuLup21eQ5r9WXhs4_qy7VTrgPI,79
-persidict-0.32.8.dist-info/METADATA,sha256=Bp5vJrqVsfXCNle3grOIArHx-l931DmhJ9NUCSs-uMY,9334
-persidict-0.32.8.dist-info/RECORD,,
+persidict/write_once_dict.py,sha256=-XHQhTEdvPHTKqLXK4WWW0k5cFitkzalVJC1n4BbKGo,6496
+persidict-0.34.2.dist-info/WHEEL,sha256=Jb20R3Ili4n9P1fcwuLup21eQ5r9WXhs4_qy7VTrgPI,79
+persidict-0.34.2.dist-info/METADATA,sha256=ihAySv6w1tA9-sHA6ONLvM0aDrRkn5RA4a1-G8dFxeE,9312
+persidict-0.34.2.dist-info/RECORD,,