persidict 0.32.3__tar.gz → 0.32.8__tar.gz

{persidict-0.32.3 → persidict-0.32.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: persidict
-Version: 0.32.3
+Version: 0.32.8
 Summary: Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud.
 Keywords: persistence,dicts,distributed,parallel
 Author: Vlad (Volodymyr) Pavlov

{persidict-0.32.3 → persidict-0.32.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "persidict"
-version = "0.32.3"
+version = "0.32.8"
 description = "Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud."
 readme = "README.md"
 requires-python = ">=3.10"

{persidict-0.32.3 → persidict-0.32.8}/src/persidict/file_dir_dict.py

@@ -103,17 +103,6 @@ class FileDirDict(PersiDict):
         self._base_dir = os.path.abspath(base_dir)
 
 
-    def __repr__(self):
-        """Return repr(self)."""
-
-        repr_str = super().__repr__()
-        repr_str = repr_str[:-1] + f", _base_dir={self._base_dir}"
-        repr_str += f", file_type={self.file_type}"
-        repr_str += " )"
-
-        return repr_str
-
-
     def get_params(self):
         """Return configuration parameters of the dictionary.
 
@@ -150,23 +139,9 @@ class FileDirDict(PersiDict):
     def __len__(self) -> int:
         """ Get the number of key-value pairs in the dictionary."""
 
-        num_files = 0
         suffix = "." + self.file_type
-        stack = [self._base_dir]
-
-        while stack:
-            path = stack.pop()
-            try:
-                with os.scandir(path) as it:
-                    for entry in it:
-                        if entry.is_dir(follow_symlinks=False):
-                            stack.append(entry.path)
-                        elif entry.is_file(follow_symlinks=False) and entry.name.endswith(suffix):
-                            num_files += 1
-            except PermissionError:
-                continue
-
-        return num_files
+        return sum(1 for _, _, files in os.walk(self._base_dir)
+                   for f in files if f.endswith(suffix))
 
 
     def clear(self) -> None:
@@ -185,6 +160,7 @@ class FileDirDict(PersiDict):
                     len(os.listdir(subdir_name)) == 0 ):
                 os.rmdir(subdir_name)
 
+
     def _build_full_path(self
                          , key:SafeStrTuple
                          , create_subdirs:bool=False
@@ -393,24 +369,28 @@ class FileDirDict(PersiDict):
         os.remove(filename)
 
 
-    def _generic_iter(self, iter_type: str):
+    def _generic_iter(self, result_type: set[str]):
         """Underlying implementation for .items()/.keys()/.values() iterators"""
-        assert iter_type in {"keys", "values", "items"}
+        assert isinstance(result_type, set)
+        assert 1 <= len(result_type) <= 3
+        assert len(result_type | {"keys", "values", "timestamps"}) == 3
+        assert 1 <= len(result_type & {"keys", "values", "timestamps"}) <= 3
+
         walk_results = os.walk(self._base_dir)
         ext_len = len(self.file_type) + 1
 
         def splitter(dir_path: str):
             """Transform a dirname into a PersiDictKey key"""
-            result = []
+            splitted_str = []
             if dir_path == ".":
-                return result
+                return splitted_str
             while True:
                 head, tail = os.path.split(dir_path)
-                result = [tail] + result
+                splitted_str = [tail] + splitted_str
                 dir_path = head
                 if len(head) == 0:
                     break
-            return tuple(result)
+            return tuple(splitted_str)
 
         def step():
             suffix = "." + self.file_type
@@ -423,14 +403,26 @@ class FileDirDict(PersiDict):
                         result_key = (*splitter(prefix_key), f[:-ext_len])
                         result_key = SafeStrTuple(result_key)
 
-                        if iter_type == "keys":
-                            yield unsign_safe_str_tuple(
+                        to_return = []
+
+                        if "keys" in result_type:
+                            key_to_return= unsign_safe_str_tuple(
                                 result_key, self.digest_len)
-                        elif iter_type == "values":
-                            yield self[result_key]
+                            to_return.append(key_to_return)
+
+                        if "values" in result_type:
+                            full_path = os.path.join(dir_name, f)
+                            value_to_return = self._read_from_file(full_path)
+                            to_return.append(value_to_return)
+
+                        if len(result_type) == 1:
+                            yield to_return[0]
                         else:
-                            yield (unsign_safe_str_tuple(
-                                result_key, self.digest_len), self[result_key])
+                            if "timestamps" in result_type:
+                                timestamp_to_return = os.path.getmtime(
+                                    os.path.join(dir_name, f))
+                                to_return.append(timestamp_to_return)
+                            yield tuple(to_return)
 
         return step()
 

{persidict-0.32.3 → persidict-0.32.8}/src/persidict/persi_dict.py

@@ -21,6 +21,7 @@ even after the Python process that created the dictionary has terminated.
 from __future__ import annotations
 
 from abc import abstractmethod
+import heapq
 import random
 from parameterizable import ParameterizableClass, sort_dict_by_keys
 from typing import Any, Sequence, Optional
@@ -87,7 +88,7 @@ class PersiDict(MutableMapping, ParameterizableClass):
                  , immutable_items:bool = False
                  , digest_len:int = 8
                  , base_class_for_values:Optional[type] = None
-                 , *args, **kwargas):
+                 , *args, **kwargs):
         self.digest_len = int(digest_len)
         if digest_len < 0:
             raise ValueError("digest_len must be non-negative")
@@ -133,13 +134,9 @@ class PersiDict(MutableMapping, ParameterizableClass):
 
     def __repr__(self) -> str:
         """Return repr(self)"""
-        repr_str = self.__class__.__name__ + "("
-        repr_str += repr(dict(self.items()))
-        repr_str += f", immutable_items={self.immutable_items}"
-        repr_str += f", digest_len={self.digest_len}"
-        repr_str += f", base_class_for_values={self.base_class_for_values}"
-        repr_str += ")"
-        return repr_str
+        params = self.get_params()
+        params_str = ', '.join(f'{k}={v!r}' for k, v in params.items())
+        return f'{self.__class__.__name__}({params_str})'
 
 
     def __str__(self) -> str:
@@ -185,30 +182,48 @@ class PersiDict(MutableMapping, ParameterizableClass):
 
 
     @abstractmethod
-    def _generic_iter(self, iter_type: str):
-        """Underlying implementation for .items()/.keys()/.values() iterators"""
-        assert iter_type in {"keys", "values", "items"}
+    def _generic_iter(self, result_type: set[str]) -> Any:
+        """Underlying implementation for items/keys/values/... iterators"""
+        assert isinstance(result_type, set)
+        assert 1 <= len(result_type) <= 3
+        assert len(result_type | {"keys", "values", "timestamps"}) == 3
+        assert 1 <= len(result_type & {"keys", "values", "timestamps"}) <= 3
         raise NotImplementedError
 
 
     def __iter__(self):
         """Implement iter(self)."""
-        return self._generic_iter("keys")
+        return self._generic_iter({"keys"})
 
 
     def keys(self):
-        """D.keys() -> iterator object that provides access to D's keys"""
-        return self._generic_iter("keys")
+        """iterator object that provides access to keys"""
+        return  self._generic_iter({"keys"})
+
+
+    def keys_and_timestamps(self):
+        """iterator object that provides access to keys and timestamps"""
+        return self._generic_iter({"keys", "timestamps"})
 
 
     def values(self):
         """D.values() -> iterator object that provides access to D's values"""
-        return self._generic_iter("values")
+        return self._generic_iter({"values"})
+
+
+    def values_and_timestamps(self):
+        """iterator object that provides access to values and timestamps"""
+        return self._generic_iter({"values", "timestamps"})
 
 
     def items(self):
         """D.items() -> iterator object that provides access to D's items"""
-        return self._generic_iter("items")
+        return self._generic_iter({"keys", "values"})
+
+
+    def items_and_timestamps(self):
+        """iterator object that provides access to keys, values, and timestamps"""
+        return self._generic_iter({"keys", "values", "timestamps"})
 
 
     def setdefault(self, key:PersiDictKey, default:Any=None) -> Any:
@@ -350,39 +365,51 @@ class PersiDict(MutableMapping, ParameterizableClass):
 
         This method is absent in the original Python dict API.
         """
-        all_keys = list(self.keys())
-        # all_keys.sort(key=lambda k: self.timestamp(k))
-        all_keys.sort(key=self.timestamp)
-        if max_n is None or max_n > len(all_keys):
-            max_n = len(all_keys)
-        result = all_keys[:max_n]
-        return result
+        if max_n is None:
+            # If we need all keys, sort them all by timestamp
+            key_timestamp_pairs = list(self.keys_and_timestamps())
+            key_timestamp_pairs.sort(key=lambda x: x[1])
+            return [key for key,_ in key_timestamp_pairs]
+        elif max_n <= 0:
+            return []
+        else:
+            # Use heapq.nsmallest for efficient partial sorting without loading all keys into memory
+            smallest_pairs = heapq.nsmallest(max_n
+                                             , self.keys_and_timestamps()
+                                             , key=lambda x: x[1])
+            return [key for key,_ in smallest_pairs]
 
 
-    def newest_keys(self, max_n=None):
-        """Return max_n the newest keys in the dictionary.
+    def oldest_values(self, max_n=None):
+        """Return max_n the oldest values in the dictionary.
 
-        If max_n is None, return all keys.
+        If max_n is None, return all values.
 
         This method is absent in the original Python dict API.
         """
-        all_keys = list(self.keys())
-        # all_keys.sort(key=lambda k: self.timestamp(k), reverse=True)
-        all_keys.sort(key=self.timestamp, reverse=True)
-        if max_n is None or max_n > len(all_keys):
-            max_n = len(all_keys)
-        result = all_keys[:max_n]
-        return result
+        return [self[k] for k in self.oldest_keys(max_n)]
 
 
-    def oldest_values(self, max_n=None):
-        """Return max_n the oldest values in the dictionary.
+    def newest_keys(self, max_n=None):
+        """Return max_n the newest keys in the dictionary.
 
-        If max_n is None, return all values.
+        If max_n is None, return all keys.
 
         This method is absent in the original Python dict API.
         """
-        return [self[k] for k in self.oldest_keys(max_n)]
+        if max_n is None:
+            # If we need all keys, sort them all by timestamp in reverse order
+            key_timestamp_pairs = list(self.keys_and_timestamps())
+            key_timestamp_pairs.sort(key=lambda x:x[1], reverse=True)
+            return [key for key,_ in key_timestamp_pairs]
+        elif max_n <= 0:
+            return []
+        else:
+            # Use heapq.nlargest for efficient partial sorting without loading all keys into memory
+            largest_pairs = heapq.nlargest(max_n
+                                            , self.keys_and_timestamps()
+                                            , key=lambda item: item[1])
+            return [key for key,_ in largest_pairs]
 
 
     def newest_values(self, max_n=None):

{persidict-0.32.3 → persidict-0.32.8}/src/persidict/s3_dict.py

@@ -96,20 +96,6 @@ class S3Dict(PersiDict):
             self.root_prefix += "/"
 
 
-    def __repr__(self) -> str:
-        """Return repr(self)."""
-
-        repr_str = super().__repr__()
-        repr_str = repr_str[:-1] + f", _base_dir={self.local_cache._base_dir}"
-        repr_str += f", file_type={self.file_type}"
-        repr_str += f", region={self.region}"
-        repr_str += f", bucket_name={self.bucket_name}"
-        repr_str += f", root_prefix={self.root_prefix}"
-        repr_str += " )"
-
-        return repr_str
-
-
     def get_params(self):
         """Return configuration parameters of the object as a dictionary.
 
@@ -269,9 +255,14 @@ class S3Dict(PersiDict):
         return num_files
 
 
-    def _generic_iter(self, iter_type: str):
+    def _generic_iter(self, result_type: str):
         """Underlying implementation for .items()/.keys()/.values() iterators"""
-        assert iter_type in {"keys", "values", "items"}
+
+        assert isinstance(result_type, set)
+        assert 1 <= len(result_type) <= 3
+        assert len(result_type | {"keys", "values", "timestamps"}) == 3
+        assert 1 <= len(result_type & {"keys", "values", "timestamps"}) <= 3
+
         suffix = "." + self.file_type
         ext_len = len(self.file_type) + 1
         prefix_len = len(self.root_prefix)
@@ -295,14 +286,25 @@ class S3Dict(PersiDict):
                     if not obj_name.endswith(suffix):
                         continue
                     obj_key = splitter(obj_name)
-                    if iter_type == "keys":
-                        yield unsign_safe_str_tuple(
+
+                    to_return = []
+
+                    if "keys" in result_type:
+                        key_to_return = unsign_safe_str_tuple(
                             obj_key, self.digest_len)
-                    elif iter_type == "values":
-                        yield self[obj_key]
+                        to_return.append(key_to_return)
+
+                    if "values" in result_type:
+                        value_to_return = self[obj_key]
+                        to_return.append(value_to_return)
+
+                    if len(result_type) == 1:
+                        yield to_return[0]
                     else:
-                        yield (unsign_safe_str_tuple(
-                            obj_key, self.digest_len), self[obj_key])
+                        if "timestamps" in result_type:
+                            timestamp_to_return = key["LastModified"].timestamp()
+                            to_return.append(timestamp_to_return)
+                        yield tuple(to_return)
 
         return step()
 

{persidict-0.32.3 → persidict-0.32.8}/src/persidict/safe_chars.py

@@ -1,12 +1,11 @@
 import string
-from copy import deepcopy
 
 SAFE_CHARS_SET = set(string.ascii_letters + string.digits + "()_-~.=")
 SAFE_STRING_MAX_LENGTH = 254
 
 def get_safe_chars() -> set[str]:
     """Return a set of allowed characters."""
-    return deepcopy(SAFE_CHARS_SET)
+    return SAFE_CHARS_SET.copy()
 
 def replace_unsafe_chars(a_str:str, replace_with:str) -> str :
     """ Replace unsafe (special) characters with allowed (safe) ones."""

{persidict-0.32.3 → persidict-0.32.8}/src/persidict/write_once_dict.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import time
+from functools import cache
 
 from deepdiff import DeepDiff
 from parameterizable import register_parameterizable_class, sort_dict_by_keys