persidict 0.32.7__tar.gz → 0.34.1__tar.gz

{persidict-0.32.7 → persidict-0.34.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: persidict
-Version: 0.32.7
+Version: 0.34.1
 Summary: Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud.
 Keywords: persistence,dicts,distributed,parallel
 Author: Vlad (Volodymyr) Pavlov
@@ -21,7 +21,6 @@ Requires-Dist: joblib
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: jsonpickle
-Requires-Dist: joblib
 Requires-Dist: deepdiff
 Requires-Dist: boto3 ; extra == 'aws'
 Requires-Dist: boto3 ; extra == 'dev'

{persidict-0.32.7 → persidict-0.34.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "persidict"
-version = "0.32.7"
+version = "0.34.1"
 description = "Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -31,7 +31,6 @@ dependencies = [
     "numpy",
     "pandas",
     "jsonpickle",
-    "joblib",
     "deepdiff"
 ]
 

{persidict-0.32.7 → persidict-0.34.1}/src/persidict/file_dir_dict.py

@@ -11,6 +11,7 @@ from __future__ import annotations
 
 import os
 import random
+import tempfile
 import time
 from typing import Any, Optional
 
@@ -79,6 +80,10 @@ class FileDirDict(PersiDict):
 
         assert file_type == replace_unsafe_chars(file_type, "")
         self.file_type = file_type
+        if self.file_type == "__etag__":
+            raise ValueError(
+                "file_type cannot be 'etag' as it is a reserved"
+                " extension for S3 caching.")
 
         if (base_class_for_values is None or
                 not issubclass(base_class_for_values,str)):
@@ -90,13 +95,7 @@ class FileDirDict(PersiDict):
         if os.path.isfile(base_dir):
             raise ValueError(f"{base_dir} is a file, not a directory.")
 
-        try: # extra protection to better handle concurrent access
-            if not os.path.isdir(base_dir):
-                os.mkdir(base_dir)
-        except:
-            time.sleep(random.random()/random.randint(1, 3))
-            if not os.path.isdir(base_dir):
-                os.mkdir(base_dir)
+        os.makedirs(base_dir, exist_ok=True)
         assert os.path.isdir(base_dir)
 
         # self.base_dir_param = _base_dir
@@ -137,25 +136,16 @@ class FileDirDict(PersiDict):
 
 
     def __len__(self) -> int:
-        """ Get the number of key-value pairs in the dictionary."""
-
-        num_files = 0
-        suffix = "." + self.file_type
-        stack = [self._base_dir]
+        """ Get the number of key-value pairs in the dictionary.
 
-        while stack:
-            path = stack.pop()
-            try:
-                with os.scandir(path) as it:
-                    for entry in it:
-                        if entry.is_dir(follow_symlinks=False):
-                            stack.append(entry.path)
-                        elif entry.is_file(follow_symlinks=False) and entry.name.endswith(suffix):
-                            num_files += 1
-            except PermissionError:
-                continue
+        WARNING: This operation can be slow on large dictionaries as it
+        needs to recursively walk the entire base directory.
+        Avoid using it in performance-sensitive code.
+        """
 
-        return num_files
+        suffix = "." + self.file_type
+        return sum(1 for _, _, files in os.walk(self._base_dir)
+                   for f in files if f.endswith(suffix))
 
 
     def clear(self) -> None:
@@ -164,6 +154,9 @@ class FileDirDict(PersiDict):
         if self.immutable_items:
             raise KeyError("Can't clear a dict that contains immutable items")
 
+        # we can't use shutil.rmtree() because
+        # there may be overlapping dictionaries
+        # with different file_type-s
         for subdir_info in os.walk(self._base_dir, topdown=False):
             (subdir_name, _, files) = subdir_info
             suffix = "." + self.file_type
@@ -174,6 +167,7 @@ class FileDirDict(PersiDict):
                     len(os.listdir(subdir_name)) == 0 ):
                 os.rmdir(subdir_name)
 
+
     def _build_full_path(self
                          , key:SafeStrTuple
                          , create_subdirs:bool=False
@@ -185,17 +179,8 @@ class FileDirDict(PersiDict):
         dir_names = key[:-1] if is_file_path else key
 
         if create_subdirs:
-            current_dir = dir_names[0]
-            for dir_name in dir_names[1:]:
-                new_dir = os.path.join(current_dir, dir_name)
-                try: # extra protection to better handle concurrent access
-                    if not os.path.isdir(new_dir):
-                        os.mkdir(new_dir)
-                except:
-                    time.sleep(random.random()/random.randint(1, 3))
-                    if not os.path.isdir(new_dir):
-                        os.mkdir(new_dir)
-                current_dir = new_dir
+            dir_path = os.path.join(*dir_names)
+            os.makedirs(dir_path, exist_ok=True)
 
         if is_file_path:
             file_name = key[-1] + "." + self.file_type
@@ -282,25 +267,50 @@ class FileDirDict(PersiDict):
         for i in range(n_retries):
             try:
                 return self._read_from_file_impl(file_name)
-            except:
-                time.sleep(random.random()/random.randint(1, 10))
-
-        return self._read_from_file_impl(file_name)
+            except Exception as e:
+                if i < n_retries - 1:
+                    time.sleep(random.uniform(0.01, 0.1) * (2 ** i))
+                else:
+                    raise e
 
 
     def _save_to_file_impl(self, file_name:str, value:Any) -> None:
         """Save a value to a file. """
 
-        if self.file_type == "pkl":
-            with open(file_name, 'wb') as f:
-                joblib.dump(value, f, compress='lz4')
-        elif self.file_type == "json":
-            with open(file_name, 'w') as f:
-                f.write(jsonpickle.dumps(value, indent=4))
-        else:
-            with open(file_name, 'w') as f:
-                f.write(value)
+        dir_name = os.path.dirname(file_name)
+        # Use a temporary file and atomic rename to prevent data corruption
+        fd, temp_path = tempfile.mkstemp(dir=dir_name, prefix=".__tmp__")
+
+        try:
+            if self.file_type == "pkl":
+                with open(fd, 'wb') as f:
+                    joblib.dump(value, f, compress='lz4')
+                    f.flush()
+                    os.fsync(f.fileno())
+            elif self.file_type == "json":
+                with open(fd, 'w') as f:
+                    f.write(jsonpickle.dumps(value, indent=4))
+                    f.flush()
+                    os.fsync(f.fileno())
+            else:
+                with open(fd, 'w') as f:
+                    f.write(value)
+                    f.flush()
+                    os.fsync(f.fileno())
+            os.replace(temp_path, file_name)
+            try:
+                if os.name == 'posix':
+                    dir_fd = os.open(dir_name, os.O_RDONLY)
+                    try:
+                        os.fsync(dir_fd)
+                    finally:
+                        os.close(dir_fd)
+            except OSError:
+                pass
 
+        except:
+            os.remove(temp_path)
+            raise
 
     def _save_to_file(self, file_name:str, value:Any) -> None:
         """Save a value to a file. """
@@ -310,16 +320,17 @@ class FileDirDict(PersiDict):
             raise ValueError("When base_class_for_values is not str,"
                 + " file_type must be pkl or json.")
 
-        n_retries = 3
+        n_retries = 8
         # extra protections to better handle concurrent writes
         for i in range(n_retries):
-            try: # extra protections to better handle concurrent writes
+            try:
                 self._save_to_file_impl(file_name, value)
                 return
-            except:
-                time.sleep(random.random()/random.randint(1, 5))
-
-        self._save_to_file_impl(file_name, value)
+            except Exception as e:
+                if i < n_retries - 1:
+                    time.sleep(random.uniform(0.01, 0.1) * (2 ** i))
+                else:
+                    raise e
 
 
     def __contains__(self, key:PersiDictKey) -> bool:
@@ -394,16 +405,9 @@ class FileDirDict(PersiDict):
 
         def splitter(dir_path: str):
             """Transform a dirname into a PersiDictKey key"""
-            splitted_str = []
             if dir_path == ".":
-                return splitted_str
-            while True:
-                head, tail = os.path.split(dir_path)
-                splitted_str = [tail] + splitted_str
-                dir_path = head
-                if len(head) == 0:
-                    break
-            return tuple(splitted_str)
+                return []
+            return dir_path.split(os.sep)
 
         def step():
             suffix = "." + self.file_type
@@ -424,7 +428,8 @@ class FileDirDict(PersiDict):
                             to_return.append(key_to_return)
 
                         if "values" in result_type:
-                            value_to_return = self[result_key]
+                            full_path = os.path.join(dir_name, f)
+                            value_to_return = self._read_from_file(full_path)
                             to_return.append(value_to_return)
 
                         if len(result_type) == 1:
@@ -451,7 +456,6 @@ class FileDirDict(PersiDict):
 
     def random_key(self) -> PersiDictKey | None:
         # canonicalise extension once
-        early_exit_cap = 10_000
         ext = None
         if self.file_type:
             ext = self.file_type.lower()
@@ -479,9 +483,6 @@ class FileDirDict(PersiDict):
                             seen += 1
                             if random.random() < 1 / seen:  # reservoir k=1
                                 winner = ent.path
-                            # early‑exit when cap reached
-                            if early_exit_cap and seen >= early_exit_cap:
-                                return self._build_key_from_full_path(os.path.abspath(winner))
             except PermissionError:
                 continue
 

{persidict-0.32.7 → persidict-0.34.1}/src/persidict/s3_dict.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
 import os
+import tempfile
 from typing import Any, Optional
 
 import boto3
+from botocore.exceptions import ClientError
+
 import parameterizable
 from parameterizable.dict_sorter import sort_dict_by_keys
 
@@ -60,16 +63,19 @@ class S3Dict(PersiDict):
         check types of values in the dictionary. If not specified,
         no type checking will be performed and all types will be allowed.
 
-        file_type is extension, which will be used for all files in the dictionary.
+        file_type is an extension, which will be used for all files in the dictionary.
         If file_type has one of two values: "lz4" or "json", it defines
         which file format will be used by FileDirDict to store values.
         For all other values of file_type, the file format will always be plain
-        text. "lz4" or "json" allow to store arbitrary Python objects,
+        text. "lz4" or "json" allow storing arbitrary Python objects,
         while all other file_type-s only work with str objects.
         """
 
         super().__init__(immutable_items = immutable_items, digest_len = 0)
         self.file_type = file_type
+        if self.file_type == "__etag__":
+            raise ValueError(
+                "file_type cannot be 'etag' as it is a reserved extension for caching.")
 
         self.local_cache = FileDirDict(
             base_dir= base_dir
@@ -152,26 +158,89 @@ class S3Dict(PersiDict):
             return False
 
 
+    def _write_etag_file(self, file_name: str, etag: str):
+        """Atomically write the ETag to its cache file."""
+        if not etag:
+            return
+        etag_file_name = file_name + ".__etag__"
+        dir_name = os.path.dirname(etag_file_name)
+        # Write to a temporary file and then rename for atomicity
+        fd, temp_path = tempfile.mkstemp(dir=dir_name)
+        try:
+            with os.fdopen(fd, "w") as f:
+                f.write(etag)
+                f.flush()
+                os.fsync(f.fileno())
+            os.replace(temp_path, etag_file_name)
+            try:
+                if os.name == 'posix':
+                    dir_fd = os.open(dir_name, os.O_RDONLY)
+                    try:
+                        os.fsync(dir_fd)
+                    finally:
+                        os.close(dir_fd)
+            except OSError:
+                pass
+        except:
+            os.remove(temp_path)
+            raise
+
+
     def __getitem__(self, key:PersiDictKey) -> Any:
         """X.__getitem__(y) is an equivalent to X[y]. """
 
         key = SafeStrTuple(key)
         file_name = self.local_cache._build_full_path(key, create_subdirs=True)
 
-        if self.immutable_items:
+        if self.immutable_items and os.path.exists(file_name):
+            return self.local_cache._read_from_file(file_name)
+
+        obj_name = self._build_full_objectname(key)
+
+
+        try:
+            head = self.s3_client.head_object(
+                Bucket=self.bucket_name, Key=obj_name)
+            s3_etag = head.get("ETag")
+        except ClientError as e:
+            if e.response['Error']['Code'] == '404':
+                raise KeyError(f"Key {key} not found in S3 bucket {self.bucket_name}")
+            else:
+                # Re-raise other client errors (e.g., permissions, throttling)
+                raise
+
+        etag_file_name = file_name + ".__etag__"
+        if not self.immutable_items and os.path.exists(file_name) and os.path.exists(etag_file_name):
+            with open(etag_file_name, "r") as f:
+                cached_etag = f.read()
+            if cached_etag == s3_etag:
+                return self.local_cache._read_from_file(file_name)
+
+        dir_name = os.path.dirname(file_name)
+        fd, temp_path = tempfile.mkstemp(dir=dir_name, prefix=".__tmp__")
+
+        try:
+            with os.fdopen(fd, 'wb') as f:
+                self.s3_client.download_fileobj(self.bucket_name, obj_name, f)
+                f.flush()
+                os.fsync(f.fileno())
+            os.replace(temp_path, file_name)
             try:
-                result = self.local_cache._read_from_file(file_name)
-                return result
-            except:
+                if os.name == 'posix':
+                    dir_fd = os.open(dir_name, os.O_RDONLY)
+                    try:
+                        os.fsync(dir_fd)
+                    finally:
+                        os.close(dir_fd)
+            except OSError:
                 pass
+        except:
+            os.remove(temp_path)  # Clean up temp file on failure
+            raise
 
-        obj_name = self._build_full_objectname(key)
-        self.s3_client.download_file(self.bucket_name, obj_name, file_name)
-        result = self.local_cache._read_from_file(file_name)
-        if not self.immutable_items:
-            os.remove(file_name)
+        self._write_etag_file(file_name, s3_etag)
 
-        return result
+        return self.local_cache._read_from_file(file_name)
 
 
     def __setitem__(self, key:PersiDictKey, value:Any):
@@ -196,28 +265,27 @@ class S3Dict(PersiDict):
                     + f"but it is {type(value)} instead." )
 
         key = SafeStrTuple(key)
-        file_name = self.local_cache._build_full_path(key, create_subdirs=True)
-        obj_name = self._build_full_objectname(key)
 
-        if self.immutable_items:
-            key_is_present = False
-            if os.path.exists(file_name):
-                key_is_present = True
-            else:
-                try:
-                    self.s3_client.head_object(
-                        Bucket=self.bucket_name, Key=obj_name)
-                    key_is_present = True
-                except:
-                    key_is_present = False
+        if self.immutable_items and key in self:
+            raise KeyError("Can't modify an immutable item")
 
-            if key_is_present:
-                raise KeyError("Can't modify an immutable item")
+        file_name = self.local_cache._build_full_path(key, create_subdirs=True)
+        obj_name = self._build_full_objectname(key)
 
         self.local_cache._save_to_file(file_name, value)
         self.s3_client.upload_file(file_name, self.bucket_name, obj_name)
-        if not self.immutable_items:
-            os.remove(file_name)
+
+        try:
+            head = self.s3_client.head_object(
+                Bucket=self.bucket_name, Key=obj_name)
+            s3_etag = head.get("ETag")
+            self._write_etag_file(file_name, s3_etag)
+        except ClientError:
+            # If we can't get ETag, we should remove any existing etag file
+            # to force a re-download on the next __getitem__ call.
+            etag_file_name = file_name + ".__etag__"
+            if os.path.exists(etag_file_name):
+                os.remove(etag_file_name)
 
 
     def __delitem__(self, key:PersiDictKey):
@@ -231,10 +299,17 @@ class S3Dict(PersiDict):
         file_name = self.local_cache._build_full_path(key)
         if os.path.isfile(file_name):
             os.remove(file_name)
-
+        etag_file_name = file_name + ".__etag__"
+        if os.path.isfile(etag_file_name):
+            os.remove(etag_file_name)
 
     def __len__(self) -> int:
-        """Return len(self). """
+        """Return len(self).
+
+        WARNING: This operation can be very slow and costly on large S3 buckets
+        as it needs to iterate over all objects in the dictionary's prefix.
+        Avoid using it in performance-sensitive code.
+        """
 
         num_files = 0
         suffix = "." + self.file_type

{persidict-0.32.7 → persidict-0.34.1}/src/persidict/safe_chars.py

@@ -1,12 +1,11 @@
 import string
-from copy import deepcopy
 
 SAFE_CHARS_SET = set(string.ascii_letters + string.digits + "()_-~.=")
 SAFE_STRING_MAX_LENGTH = 254
 
 def get_safe_chars() -> set[str]:
     """Return a set of allowed characters."""
-    return deepcopy(SAFE_CHARS_SET)
+    return SAFE_CHARS_SET.copy()
 
 def replace_unsafe_chars(a_str:str, replace_with:str) -> str :
     """ Replace unsafe (special) characters with allowed (safe) ones."""

{persidict-0.32.7 → persidict-0.34.1}/src/persidict/safe_str_tuple.py

@@ -43,7 +43,7 @@ class SafeStrTuple(Sequence, Hashable):
             elif isinstance(a, str):
                 assert len(a) > 0
                 assert len(a) < SAFE_STRING_MAX_LENGTH
-                assert len(set(a) - SAFE_CHARS_SET) == 0
+                assert all(c in SAFE_CHARS_SET for c in a)
                 candidate_strings.append(a)
             elif _is_sequence_not_mapping(a):
                 if len(a) > 0:

{persidict-0.32.7 → persidict-0.34.1}/src/persidict/write_once_dict.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import time
+from functools import cache
 
 from deepdiff import DeepDiff
 from parameterizable import register_parameterizable_class, sort_dict_by_keys
@@ -115,17 +116,19 @@ class WriteOnceDict(PersiDict):
         """
         check_needed = False
 
-        try: # extra protections to better handle concurrent writes
-            if key in self._wrapped_dict:
-                check_needed = True
-            else:
-                self._wrapped_dict[key] = value
-        except:
-            time.sleep(random.random()/random.randint(1,5))
-            if key in self._wrapped_dict:
-                check_needed = True
-            else:
-                self._wrapped_dict[key] = value
+        n_retries = 8
+        for i in range(n_retries):
+            try:  # extra protections to better handle concurrent writes
+                if key in self._wrapped_dict:
+                    check_needed = True
+                else:
+                    self._wrapped_dict[key] = value
+                break
+            except Exception as e:
+                if i < n_retries - 1:
+                    time.sleep(random.uniform(0.01, 0.1) * (2 ** i))
+                else:
+                    raise e
 
         if not key in self._wrapped_dict:
             raise KeyError(