persidict 0.32.2__py3-none-any.whl → 0.32.7__py3-none-any.whl

persidict/file_dir_dict.py

@@ -103,17 +103,6 @@ class FileDirDict(PersiDict):
         self._base_dir = os.path.abspath(base_dir)
 
 
-    def __repr__(self):
-        """Return repr(self)."""
-
-        repr_str = super().__repr__()
-        repr_str = repr_str[:-1] + f", _base_dir={self._base_dir}"
-        repr_str += f", file_type={self.file_type}"
-        repr_str += " )"
-
-        return repr_str
-
-
     def get_params(self):
         """Return configuration parameters of the dictionary.
 
@@ -393,24 +382,28 @@ class FileDirDict(PersiDict):
         os.remove(filename)
 
 
-    def _generic_iter(self, iter_type: str):
+    def _generic_iter(self, result_type: set[str]):
         """Underlying implementation for .items()/.keys()/.values() iterators"""
-        assert iter_type in {"keys", "values", "items"}
+        assert isinstance(result_type, set)
+        assert 1 <= len(result_type) <= 3
+        assert len(result_type | {"keys", "values", "timestamps"}) == 3
+        assert 1 <= len(result_type & {"keys", "values", "timestamps"}) <= 3
+
         walk_results = os.walk(self._base_dir)
         ext_len = len(self.file_type) + 1
 
         def splitter(dir_path: str):
             """Transform a dirname into a PersiDictKey key"""
-            result = []
+            splitted_str = []
             if dir_path == ".":
-                return result
+                return splitted_str
             while True:
                 head, tail = os.path.split(dir_path)
-                result = [tail] + result
+                splitted_str = [tail] + splitted_str
                 dir_path = head
                 if len(head) == 0:
                     break
-            return tuple(result)
+            return tuple(splitted_str)
 
         def step():
             suffix = "." + self.file_type
@@ -423,14 +416,25 @@ class FileDirDict(PersiDict):
                         result_key = (*splitter(prefix_key), f[:-ext_len])
                         result_key = SafeStrTuple(result_key)
 
-                        if iter_type == "keys":
-                            yield unsign_safe_str_tuple(
+                        to_return = []
+
+                        if "keys" in result_type:
+                            key_to_return= unsign_safe_str_tuple(
                                 result_key, self.digest_len)
-                        elif iter_type == "values":
-                            yield self[result_key]
+                            to_return.append(key_to_return)
+
+                        if "values" in result_type:
+                            value_to_return = self[result_key]
+                            to_return.append(value_to_return)
+
+                        if len(result_type) == 1:
+                            yield to_return[0]
                         else:
-                            yield (unsign_safe_str_tuple(
-                                result_key, self.digest_len), self[result_key])
+                            if "timestamps" in result_type:
+                                timestamp_to_return = os.path.getmtime(
+                                    os.path.join(dir_name, f))
+                                to_return.append(timestamp_to_return)
+                            yield tuple(to_return)
 
         return step()
 

persidict/persi_dict.py

@@ -21,6 +21,7 @@ even after the Python process that created the dictionary has terminated.
 from __future__ import annotations
 
 from abc import abstractmethod
+import heapq
 import random
 from parameterizable import ParameterizableClass, sort_dict_by_keys
 from typing import Any, Sequence, Optional
@@ -87,7 +88,7 @@ class PersiDict(MutableMapping, ParameterizableClass):
                  , immutable_items:bool = False
                  , digest_len:int = 8
                  , base_class_for_values:Optional[type] = None
-                 , *args, **kwargas):
+                 , *args, **kwargs):
         self.digest_len = int(digest_len)
         if digest_len < 0:
             raise ValueError("digest_len must be non-negative")
@@ -133,13 +134,9 @@ class PersiDict(MutableMapping, ParameterizableClass):
 
     def __repr__(self) -> str:
         """Return repr(self)"""
-        repr_str = self.__class__.__name__ + "("
-        repr_str += repr(dict(self.items()))
-        repr_str += f", immutable_items={self.immutable_items}"
-        repr_str += f", digest_len={self.digest_len}"
-        repr_str += f", base_class_for_values={self.base_class_for_values}"
-        repr_str += ")"
-        return repr_str
+        params = self.get_params()
+        params_str = ', '.join(f'{k}={v!r}' for k, v in params.items())
+        return f'{self.__class__.__name__}({params_str})'
 
 
     def __str__(self) -> str:
@@ -185,30 +182,48 @@ class PersiDict(MutableMapping, ParameterizableClass):
 
 
     @abstractmethod
-    def _generic_iter(self, iter_type: str):
-        """Underlying implementation for .items()/.keys()/.values() iterators"""
-        assert iter_type in {"keys", "values", "items"}
+    def _generic_iter(self, result_type: set[str]) -> Any:
+        """Underlying implementation for items/keys/values/... iterators"""
+        assert isinstance(result_type, set)
+        assert 1 <= len(result_type) <= 3
+        assert len(result_type | {"keys", "values", "timestamps"}) == 3
+        assert 1 <= len(result_type & {"keys", "values", "timestamps"}) <= 3
         raise NotImplementedError
 
 
     def __iter__(self):
         """Implement iter(self)."""
-        return self._generic_iter("keys")
+        return self._generic_iter({"keys"})
 
 
     def keys(self):
-        """D.keys() -> iterator object that provides access to D's keys"""
-        return self._generic_iter("keys")
+        """iterator object that provides access to keys"""
+        return  self._generic_iter({"keys"})
+
+
+    def keys_and_timestamps(self):
+        """iterator object that provides access to keys and timestamps"""
+        return self._generic_iter({"keys", "timestamps"})
 
 
     def values(self):
         """D.values() -> iterator object that provides access to D's values"""
-        return self._generic_iter("values")
+        return self._generic_iter({"values"})
+
+
+    def values_and_timestamps(self):
+        """iterator object that provides access to values and timestamps"""
+        return self._generic_iter({"values", "timestamps"})
 
 
     def items(self):
         """D.items() -> iterator object that provides access to D's items"""
-        return self._generic_iter("items")
+        return self._generic_iter({"keys", "values"})
+
+
+    def items_and_timestamps(self):
+        """iterator object that provides access to keys, values, and timestamps"""
+        return self._generic_iter({"keys", "values", "timestamps"})
 
 
     def setdefault(self, key:PersiDictKey, default:Any=None) -> Any:
@@ -350,39 +365,51 @@ class PersiDict(MutableMapping, ParameterizableClass):
 
         This method is absent in the original Python dict API.
         """
-        all_keys = list(self.keys())
-        # all_keys.sort(key=lambda k: self.timestamp(k))
-        all_keys.sort(key=self.timestamp)
-        if max_n is None or max_n > len(all_keys):
-            max_n = len(all_keys)
-        result = all_keys[:max_n]
-        return result
+        if max_n is None:
+            # If we need all keys, sort them all by timestamp
+            key_timestamp_pairs = list(self.keys_and_timestamps())
+            key_timestamp_pairs.sort(key=lambda x: x[1])
+            return [key for key,_ in key_timestamp_pairs]
+        elif max_n <= 0:
+            return []
+        else:
+            # Use heapq.nsmallest for efficient partial sorting without loading all keys into memory
+            smallest_pairs = heapq.nsmallest(max_n
+                                             , self.keys_and_timestamps()
+                                             , key=lambda x: x[1])
+            return [key for key,_ in smallest_pairs]
 
 
-    def newest_keys(self, max_n=None):
-        """Return max_n the newest keys in the dictionary.
+    def oldest_values(self, max_n=None):
+        """Return max_n the oldest values in the dictionary.
 
-        If max_n is None, return all keys.
+        If max_n is None, return all values.
 
         This method is absent in the original Python dict API.
         """
-        all_keys = list(self.keys())
-        # all_keys.sort(key=lambda k: self.timestamp(k), reverse=True)
-        all_keys.sort(key=self.timestamp, reverse=True)
-        if max_n is None or max_n > len(all_keys):
-            max_n = len(all_keys)
-        result = all_keys[:max_n]
-        return result
+        return [self[k] for k in self.oldest_keys(max_n)]
 
 
-    def oldest_values(self, max_n=None):
-        """Return max_n the oldest values in the dictionary.
+    def newest_keys(self, max_n=None):
+        """Return max_n the newest keys in the dictionary.
 
-        If max_n is None, return all values.
+        If max_n is None, return all keys.
 
         This method is absent in the original Python dict API.
         """
-        return [self[k] for k in self.oldest_keys(max_n)]
+        if max_n is None:
+            # If we need all keys, sort them all by timestamp in reverse order
+            key_timestamp_pairs = list(self.keys_and_timestamps())
+            key_timestamp_pairs.sort(key=lambda x:x[1], reverse=True)
+            return [key for key,_ in key_timestamp_pairs]
+        elif max_n <= 0:
+            return []
+        else:
+            # Use heapq.nlargest for efficient partial sorting without loading all keys into memory
+            largest_pairs = heapq.nlargest(max_n
+                                            , self.keys_and_timestamps()
+                                            , key=lambda item: item[1])
+            return [key for key,_ in largest_pairs]
 
 
     def newest_values(self, max_n=None):

persidict/s3_dict.py

@@ -96,20 +96,6 @@ class S3Dict(PersiDict):
             self.root_prefix += "/"
 
 
-    def __repr__(self) -> str:
-        """Return repr(self)."""
-
-        repr_str = super().__repr__()
-        repr_str = repr_str[:-1] + f", _base_dir={self.local_cache._base_dir}"
-        repr_str += f", file_type={self.file_type}"
-        repr_str += f", region={self.region}"
-        repr_str += f", bucket_name={self.bucket_name}"
-        repr_str += f", root_prefix={self.root_prefix}"
-        repr_str += " )"
-
-        return repr_str
-
-
     def get_params(self):
         """Return configuration parameters of the object as a dictionary.
 
@@ -253,23 +239,30 @@ class S3Dict(PersiDict):
         num_files = 0
         suffix = "." + self.file_type
 
-        paginator = self.s3_client.get_paginator("list_objects")
+        paginator = self.s3_client.get_paginator("list_objects_v2")
         page_iterator = paginator.paginate(
             Bucket=self.bucket_name, Prefix = self.root_prefix)
 
         for page in page_iterator:
-            if "Contents" in page:
-                for key in page["Contents"]:
-                    obj_name = key["Key"]
-                    if obj_name.endswith(suffix):
-                        num_files += 1
+            contents = page.get("Contents")
+            if not contents:
+                continue
+            for key in contents:
+                obj_name = key["Key"]
+                if obj_name.endswith(suffix):
+                    num_files += 1
 
         return num_files
 
 
-    def _generic_iter(self, iter_type: str):
+    def _generic_iter(self, result_type: str):
         """Underlying implementation for .items()/.keys()/.values() iterators"""
-        assert iter_type in {"keys", "values", "items"}
+
+        assert isinstance(result_type, set)
+        assert 1 <= len(result_type) <= 3
+        assert len(result_type | {"keys", "values", "timestamps"}) == 3
+        assert 1 <= len(result_type & {"keys", "values", "timestamps"}) <= 3
+
         suffix = "." + self.file_type
         ext_len = len(self.file_type) + 1
         prefix_len = len(self.root_prefix)
@@ -280,25 +273,38 @@ class S3Dict(PersiDict):
             return SafeStrTuple(result)
 
         def step():
-            paginator = self.s3_client.get_paginator("list_objects")
+            paginator = self.s3_client.get_paginator("list_objects_v2")
             page_iterator = paginator.paginate(
                 Bucket=self.bucket_name, Prefix = self.root_prefix)
 
             for page in page_iterator:
-                if "Contents" in page:
-                    for key in page["Contents"]:
-                        obj_name = key["Key"]
-                        if not obj_name.endswith(suffix):
-                            continue
-                        obj_key = splitter(obj_name)
-                        if iter_type == "keys":
-                            yield unsign_safe_str_tuple(
-                                obj_key, self.digest_len)
-                        elif iter_type == "values":
-                            yield self[obj_key]
-                        else:
-                            yield (unsign_safe_str_tuple(
-                                obj_key, self.digest_len), self[obj_key])
+                contents = page.get("Contents")
+                if not contents:
+                    continue
+                for key in contents:
+                    obj_name = key["Key"]
+                    if not obj_name.endswith(suffix):
+                        continue
+                    obj_key = splitter(obj_name)
+
+                    to_return = []
+
+                    if "keys" in result_type:
+                        key_to_return = unsign_safe_str_tuple(
+                            obj_key, self.digest_len)
+                        to_return.append(key_to_return)
+
+                    if "values" in result_type:
+                        value_to_return = self[obj_key]
+                        to_return.append(value_to_return)
+
+                    if len(result_type) == 1:
+                        yield to_return[0]
+                    else:
+                        if "timestamps" in result_type:
+                            timestamp_to_return = key["LastModified"].timestamp()
+                            to_return.append(timestamp_to_return)
+                        yield tuple(to_return)
 
         return step()
 

{persidict-0.32.2.dist-info → persidict-0.32.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: persidict
-Version: 0.32.2
+Version: 0.32.7
 Summary: Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud.
 Keywords: persistence,dicts,distributed,parallel
 Author: Vlad (Volodymyr) Pavlov

persidict-0.32.7.dist-info/RECORD

@@ -0,0 +1,14 @@
+persidict/.DS_Store,sha256=1lFlJ5EFymdzGAUAaI30vcaaLHt3F1LwpG7xILf9jsM,6148
+persidict/__init__.py,sha256=CDOSJGgCnyRTkGUTzaeg3Cqsxwx0-0EFieOtldXwAls,1380
+persidict/file_dir_dict.py,sha256=3uoAua47pIJwpCPkXtF2dIskHm73kZogy15yknLfLUk,17939
+persidict/jokers.py,sha256=kX4bE-jKWTM2ki7JOmm_2uJS8zm8u6InZ_V12xo2ImI,1436
+persidict/overlapping_multi_dict.py,sha256=a-lUbmY15_HrDq6jSIt8F8tJboqbeYiuRQeW4elf_oU,2663
+persidict/persi_dict.py,sha256=SF6aWs6kCeeW-bZ9HJwx0sPX7Xav_aURqeSZ-j5quv0,14266
+persidict/s3_dict.py,sha256=awEIu7ehh4FgKircC10lcHp86xvUctXA_6jxUmC3Hy8,12480
+persidict/safe_chars.py,sha256=WaIOliBdLlZ12CepEa1C3b1VPxvX4s2hvnZanCxM6BQ,565
+persidict/safe_str_tuple.py,sha256=cTk5BL3r-yE62EKf7VngTaUpZAdsATJPIjiCGqQzkyU,3717
+persidict/safe_str_tuple_signing.py,sha256=5uCjAVZRqOou-KpDZw-Exboc3-3vuayJMqrrt8aZ0ck,3742
+persidict/write_once_dict.py,sha256=dqzA3amVDK9D8ygU8NFT5B87smAHYWdEn78Fssj6s_g,6396
+persidict-0.32.7.dist-info/WHEEL,sha256=Jb20R3Ili4n9P1fcwuLup21eQ5r9WXhs4_qy7VTrgPI,79
+persidict-0.32.7.dist-info/METADATA,sha256=MG-CqgwJx0yz3aounX3ZT-dvD-9s2HdTzTDo9mQXN14,9334
+persidict-0.32.7.dist-info/RECORD,,

{persidict-0.32.2.dist-info → persidict-0.32.7.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: uv 0.7.19
+Generator: uv 0.8.15
 Root-Is-Purelib: true
 Tag: py3-none-any

persidict-0.32.2.dist-info/RECORD

@@ -1,13 +0,0 @@
-persidict/__init__.py,sha256=0833922468029f2453906513cda7a0dc2aacc70c74fb410589e3ad95d5f0025b,1380
-persidict/file_dir_dict.py,sha256=0c650ba2d5c17ab65ea5dd5247544f68c77e3027b022876b18ea19ade51d78a6,17527
-persidict/jokers.py,sha256=917e1b13e8ca593336922ec93a69bfdae252f339bcbba22767f575db1a362262,1436
-persidict/overlapping_multi_dict.py,sha256=6be9546e6635e7f1eb0eaea3488b7c17cb496e8a9b7988ae450796e1e95ffe85,2663
-persidict/persi_dict.py,sha256=e4f962437df50e0422d58057ab23446802a916d46afd63e69c9fe9196f12f7e8,12900
-persidict/s3_dict.py,sha256=0d8942037cee409d8bda8f63f923a8385f4f9f1fac03e765ec5f5fc675334ae2,12292
-persidict/safe_chars.py,sha256=59a20e96205d2e5675d827a911ad42ddbd553f1bd7e2cda1be765a9c2c4ce814,565
-persidict/safe_str_tuple.py,sha256=71393904bdebfb213ad8429fed59e04da52964076c01324f2238821aa4339325,3717
-persidict/safe_str_tuple_signing.py,sha256=e6e0a3015651a8ea2ef8aa43670f84c5ba1cdfedefb9ac8932aaebb7c699d1c9,3742
-persidict/write_once_dict.py,sha256=76acc0dda9950caf43f32814f0d153e41f3bb260076167449fbf05b2c8fab3f8,6396
-persidict-0.32.2.dist-info/WHEEL,sha256=607c46fee47e440c91332c738096ff0f5e54ca3b0818ee85462dd5172a38e793,79
-persidict-0.32.2.dist-info/METADATA,sha256=54072df5d3f92135b1ff868e96bf0de7a406185f243090b3725c0b5d46acf736,9334
-persidict-0.32.2.dist-info/RECORD,,