persidict 0.31.1__py3-none-any.whl → 0.32.1__py3-none-any.whl

persidict/file_dir_dict.py

@@ -152,11 +152,20 @@ class FileDirDict(PersiDict):
 
         num_files = 0
         suffix = "." + self.file_type
-        for subdir_info in os.walk(self._base_dir):
-            files = subdir_info[2]
-            files = [f_name for f_name in files
-                     if f_name.endswith(suffix)]
-            num_files += len(files)
+        stack = [self._base_dir]
+
+        while stack:
+            path = stack.pop()
+            try:
+                with os.scandir(path) as it:
+                    for entry in it:
+                        if entry.is_dir(follow_symlinks=False):
+                            stack.append(entry.path)
+                        elif entry.is_file(follow_symlinks=False) and entry.name.endswith(suffix):
+                            num_files += 1
+            except PermissionError:
+                continue
+
         return num_files
 
 
@@ -203,7 +212,39 @@ class FileDirDict(PersiDict):
             file_name = key[-1] + "." + self.file_type
             return os.path.join(*dir_names, file_name)
         else:
-            return os.path.join(*dir_names)
+            return str(os.path.join(*dir_names))
+
+
+    def _build_key_from_full_path(self, full_path:str)->SafeStrTuple:
+        """Convert a filesystem path back into a key."""
+
+        # Ensure we're working with absolute paths
+        full_path = os.path.abspath(full_path)
+
+        # Remove the base directory from the path
+        if not full_path.startswith(self._base_dir):
+            raise ValueError(f"Path {full_path} is not within base directory {self._base_dir}")
+
+        # Get the relative path
+        rel_path = full_path[len(self._base_dir):].lstrip(os.sep)
+
+        if not rel_path:
+            return SafeStrTuple()
+
+        # Split the path into components
+        path_components = rel_path.split(os.sep)
+
+        # If it's a file path, remove the file extension from the last component
+        if os.path.isfile(full_path) and path_components[-1].endswith("." + self.file_type):
+            path_components[-1] = path_components[-1][:-len("." + self.file_type)]
+
+        # Create a SafeStrTuple from the path components
+        key = SafeStrTuple(*path_components)
+
+        # Unsign the key
+        key = unsign_safe_str_tuple(key, self.digest_len)
+
+        return key
 
 
     def get_subdict(self, key:PersiDictKey) -> FileDirDict:
@@ -404,4 +445,46 @@ class FileDirDict(PersiDict):
         return os.path.getmtime(filename)
 
 
-parameterizable.register_parameterizable_class(FileDirDict)
+    def random_key(self) -> PersiDictKey | None:
+        # canonicalise extension once
+        early_exit_cap = 10_000
+        ext = None
+        if self.file_type:
+            ext = self.file_type.lower()
+            if not ext.startswith("."):
+                ext = "." + ext
+
+        stack = [self._base_dir]
+        winner: Optional[str] = None
+        seen = 0
+
+        while stack:
+            path = stack.pop()
+            try:
+                with os.scandir(path) as it:
+                    for ent in it:
+                        if ent.is_dir(follow_symlinks=False):
+                            stack.append(ent.path)
+                            continue
+
+                        # cheap name test before stat()
+                        if ext and not ent.name.lower().endswith(ext):
+                            continue
+
+                        if ent.is_file(follow_symlinks=False):
+                            seen += 1
+                            if random.random() < 1 / seen:  # reservoir k=1
+                                winner = ent.path
+                            # early‑exit when cap reached
+                            if early_exit_cap and seen >= early_exit_cap:
+                                return self._build_key_from_full_path(os.path.abspath(winner))
+            except PermissionError:
+                continue
+
+        if winner is None:
+            return None
+        else:
+            return self._build_key_from_full_path(os.path.abspath(winner))
+
+
+parameterizable.register_parameterizable_class(FileDirDict)

persidict/persi_dict.py

@@ -302,15 +302,33 @@ class PersiDict(MutableMapping, ParameterizableClass):
         return result_subdicts
 
 
-    def random_keys(self, max_n:int):
-        """Return a list of random keys from the dictionary.
+    def random_key(self) -> PersiDictKey | None:
+        """Return a random key from the dictionary.
+
+        Returns a single random key if the dictionary is not empty.
+        Returns None if the dictionary is empty.
 
         This method is absent in the original Python dict API.
+
+        Implementation uses reservoir sampling to select a uniformly random key
+        in streaming time, without loading all keys into memory or using len().
         """
-        all_keys = list(self.keys())
-        if max_n > len(all_keys):
-            max_n = len(all_keys)
-        result = random.sample(all_keys, max_n)
+        iterator = iter(self.keys())
+        try:
+            # Get the first key
+            result = next(iterator)
+        except StopIteration:
+            # Dictionary is empty
+            return None
+
+        # Reservoir sampling algorithm
+        i = 2
+        for key in iterator:
+            # Select current key with probability 1/i
+            if random.random() < 1/i:
+                result = key
+            i += 1
+
         return result
 
 
@@ -372,4 +390,4 @@ class PersiDict(MutableMapping, ParameterizableClass):
 
         This method is absent in the original Python dict API.
         """
-        return [self[k] for k in self.newest_keys(max_n)]
+        return [self[k] for k in self.newest_keys(max_n)]

{persidict-0.31.1.dist-info → persidict-0.32.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: persidict
-Version: 0.31.1
+Version: 0.32.1
 Summary: Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud.
 Keywords: persistence,dicts,distributed,parallel
 Author: Vlad (Volodymyr) Pavlov
@@ -15,12 +15,14 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Dist: parameterizable
 Requires-Dist: lz4
 Requires-Dist: joblib
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: jsonpickle
-Requires-Dist: parameterizable
+Requires-Dist: joblib
+Requires-Dist: deepdiff
 Requires-Dist: boto3 ; extra == 'aws'
 Requires-Dist: boto3 ; extra == 'dev'
 Requires-Dist: moto ; extra == 'dev'
@@ -37,7 +39,7 @@ Simple persistent dictionaries for Python.
 
 ## What Is It?
 
-`persidict` offers a simple persistent key-value store for Python. 
+`persidict` offers a very simple persistent key-value store for Python. 
 It saves the content of the dictionary in a folder on a disk 
 or in an S3 bucket on AWS. Each value is stored as a separate file / S3 object.
 Only text strings or sequences of strings are allowed as keys.

{persidict-0.31.1.dist-info → persidict-0.32.1.dist-info}/RECORD

@@ -1,14 +1,13 @@
-persidict/.DS_Store,sha256=d65165279105ca6773180500688df4bdc69a2c7b771752f0a46ef120b7fd8ec3,6148
 persidict/__init__.py,sha256=0833922468029f2453906513cda7a0dc2aacc70c74fb410589e3ad95d5f0025b,1380
-persidict/file_dir_dict.py,sha256=9587105acda4d148e25562510703573f21cff343a4757b6983f41ed4a5e9f0fb,14600
+persidict/file_dir_dict.py,sha256=0c650ba2d5c17ab65ea5dd5247544f68c77e3027b022876b18ea19ade51d78a6,17527
 persidict/jokers.py,sha256=917e1b13e8ca593336922ec93a69bfdae252f339bcbba22767f575db1a362262,1436
 persidict/overlapping_multi_dict.py,sha256=6be9546e6635e7f1eb0eaea3488b7c17cb496e8a9b7988ae450796e1e95ffe85,2663
-persidict/persi_dict.py,sha256=ea04f5ae68544a18071b9ac95a1a9bb71cf7b04dcfa25d61c0a92d4c2375d4d4,12223
+persidict/persi_dict.py,sha256=f9c290b7574312f2b536848f9d0fb6bd960c5fbea7f7615407ac37590441c5d5,12782
 persidict/s3_dict.py,sha256=0d8942037cee409d8bda8f63f923a8385f4f9f1fac03e765ec5f5fc675334ae2,12292
 persidict/safe_chars.py,sha256=59a20e96205d2e5675d827a911ad42ddbd553f1bd7e2cda1be765a9c2c4ce814,565
 persidict/safe_str_tuple.py,sha256=71393904bdebfb213ad8429fed59e04da52964076c01324f2238821aa4339325,3717
 persidict/safe_str_tuple_signing.py,sha256=e6e0a3015651a8ea2ef8aa43670f84c5ba1cdfedefb9ac8932aaebb7c699d1c9,3742
 persidict/write_once_dict.py,sha256=76acc0dda9950caf43f32814f0d153e41f3bb260076167449fbf05b2c8fab3f8,6396
-persidict-0.31.1.dist-info/WHEEL,sha256=c133ef911c90b05f7e14d8679ba99146f9154fcd271b7398cf8f672283b94e05,79
-persidict-0.31.1.dist-info/METADATA,sha256=ff478f3098ee316b0efd01678494992d84acc354fe7e5cf081686dd33c1b6a55,9283
-persidict-0.31.1.dist-info/RECORD,,
+persidict-0.32.1.dist-info/WHEEL,sha256=7de84e261f5edc1201bd668234a371ec3eb94db037d1f75c0139430c1319ab31,79
+persidict-0.32.1.dist-info/METADATA,sha256=8ba566bc5722b8e061a45d62344f2b612c9b4507d7a087351f966c8d782695dc,9334
+persidict-0.32.1.dist-info/RECORD,,

{persidict-0.31.1.dist-info → persidict-0.32.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: uv 0.7.13
+Generator: uv 0.7.16
 Root-Is-Purelib: true
 Tag: py3-none-any