persidict 0.31.1__tar.gz → 0.32.1__tar.gz

{persidict-0.31.1 → persidict-0.32.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: persidict
-Version: 0.31.1
+Version: 0.32.1
 Summary: Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud.
 Keywords: persistence,dicts,distributed,parallel
 Author: Vlad (Volodymyr) Pavlov
@@ -15,12 +15,14 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Dist: parameterizable
 Requires-Dist: lz4
 Requires-Dist: joblib
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: jsonpickle
-Requires-Dist: parameterizable
+Requires-Dist: joblib
+Requires-Dist: deepdiff
 Requires-Dist: boto3 ; extra == 'aws'
 Requires-Dist: boto3 ; extra == 'dev'
 Requires-Dist: moto ; extra == 'dev'
@@ -37,7 +39,7 @@ Simple persistent dictionaries for Python.
 
 ## What Is It?
 
-`persidict` offers a simple persistent key-value store for Python. 
+`persidict` offers a very simple persistent key-value store for Python. 
 It saves the content of the dictionary in a folder on a disk 
 or in an S3 bucket on AWS. Each value is stored as a separate file / S3 object.
 Only text strings or sequences of strings are allowed as keys.

{persidict-0.31.1 → persidict-0.32.1}/README.md

@@ -4,7 +4,7 @@ Simple persistent dictionaries for Python.
 
 ## What Is It?
 
-`persidict` offers a simple persistent key-value store for Python. 
+`persidict` offers a very simple persistent key-value store for Python. 
 It saves the content of the dictionary in a folder on a disk 
 or in an S3 bucket on AWS. Each value is stored as a separate file / S3 object.
 Only text strings or sequences of strings are allowed as keys.

{persidict-0.31.1 → persidict-0.32.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "persidict"
-version = "0.31.1"
+version = "0.32.1"
 description = "Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -25,12 +25,14 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules"
 ]
 dependencies = [
+    "parameterizable",
     "lz4",
     "joblib",
     "numpy",
     "pandas",
     "jsonpickle",
-    "parameterizable"
+    "joblib",
+    "deepdiff"
 ]
 
 [project.urls]
@@ -45,4 +47,4 @@ dev = [
 
 aws = [
     "boto3"
-]
+]

{persidict-0.31.1 → persidict-0.32.1}/src/persidict/file_dir_dict.py

@@ -152,11 +152,20 @@ class FileDirDict(PersiDict):
 
         num_files = 0
         suffix = "." + self.file_type
-        for subdir_info in os.walk(self._base_dir):
-            files = subdir_info[2]
-            files = [f_name for f_name in files
-                     if f_name.endswith(suffix)]
-            num_files += len(files)
+        stack = [self._base_dir]
+
+        while stack:
+            path = stack.pop()
+            try:
+                with os.scandir(path) as it:
+                    for entry in it:
+                        if entry.is_dir(follow_symlinks=False):
+                            stack.append(entry.path)
+                        elif entry.is_file(follow_symlinks=False) and entry.name.endswith(suffix):
+                            num_files += 1
+            except PermissionError:
+                continue
+
         return num_files
 
 
@@ -203,7 +212,39 @@ class FileDirDict(PersiDict):
             file_name = key[-1] + "." + self.file_type
             return os.path.join(*dir_names, file_name)
         else:
-            return os.path.join(*dir_names)
+            return str(os.path.join(*dir_names))
+
+
+    def _build_key_from_full_path(self, full_path:str)->SafeStrTuple:
+        """Convert a filesystem path back into a key."""
+
+        # Ensure we're working with absolute paths
+        full_path = os.path.abspath(full_path)
+
+        # Remove the base directory from the path
+        if not full_path.startswith(self._base_dir):
+            raise ValueError(f"Path {full_path} is not within base directory {self._base_dir}")
+
+        # Get the relative path
+        rel_path = full_path[len(self._base_dir):].lstrip(os.sep)
+
+        if not rel_path:
+            return SafeStrTuple()
+
+        # Split the path into components
+        path_components = rel_path.split(os.sep)
+
+        # If it's a file path, remove the file extension from the last component
+        if os.path.isfile(full_path) and path_components[-1].endswith("." + self.file_type):
+            path_components[-1] = path_components[-1][:-len("." + self.file_type)]
+
+        # Create a SafeStrTuple from the path components
+        key = SafeStrTuple(*path_components)
+
+        # Unsign the key
+        key = unsign_safe_str_tuple(key, self.digest_len)
+
+        return key
 
 
     def get_subdict(self, key:PersiDictKey) -> FileDirDict:
@@ -404,4 +445,46 @@ class FileDirDict(PersiDict):
         return os.path.getmtime(filename)
 
 
-parameterizable.register_parameterizable_class(FileDirDict)
+    def random_key(self) -> PersiDictKey | None:
+        # canonicalise extension once
+        early_exit_cap = 10_000
+        ext = None
+        if self.file_type:
+            ext = self.file_type.lower()
+            if not ext.startswith("."):
+                ext = "." + ext
+
+        stack = [self._base_dir]
+        winner: Optional[str] = None
+        seen = 0
+
+        while stack:
+            path = stack.pop()
+            try:
+                with os.scandir(path) as it:
+                    for ent in it:
+                        if ent.is_dir(follow_symlinks=False):
+                            stack.append(ent.path)
+                            continue
+
+                        # cheap name test before stat()
+                        if ext and not ent.name.lower().endswith(ext):
+                            continue
+
+                        if ent.is_file(follow_symlinks=False):
+                            seen += 1
+                            if random.random() < 1 / seen:  # reservoir k=1
+                                winner = ent.path
+                            # early‑exit when cap reached
+                            if early_exit_cap and seen >= early_exit_cap:
+                                return self._build_key_from_full_path(os.path.abspath(winner))
+            except PermissionError:
+                continue
+
+        if winner is None:
+            return None
+        else:
+            return self._build_key_from_full_path(os.path.abspath(winner))
+
+
+parameterizable.register_parameterizable_class(FileDirDict)

{persidict-0.31.1 → persidict-0.32.1}/src/persidict/persi_dict.py

@@ -302,15 +302,33 @@ class PersiDict(MutableMapping, ParameterizableClass):
         return result_subdicts
 
 
-    def random_keys(self, max_n:int):
-        """Return a list of random keys from the dictionary.
+    def random_key(self) -> PersiDictKey | None:
+        """Return a random key from the dictionary.
+
+        Returns a single random key if the dictionary is not empty.
+        Returns None if the dictionary is empty.
 
         This method is absent in the original Python dict API.
+
+        Implementation uses reservoir sampling to select a uniformly random key
+        in streaming time, without loading all keys into memory or using len().
         """
-        all_keys = list(self.keys())
-        if max_n > len(all_keys):
-            max_n = len(all_keys)
-        result = random.sample(all_keys, max_n)
+        iterator = iter(self.keys())
+        try:
+            # Get the first key
+            result = next(iterator)
+        except StopIteration:
+            # Dictionary is empty
+            return None
+
+        # Reservoir sampling algorithm
+        i = 2
+        for key in iterator:
+            # Select current key with probability 1/i
+            if random.random() < 1/i:
+                result = key
+            i += 1
+
         return result
 
 
@@ -372,4 +390,4 @@ class PersiDict(MutableMapping, ParameterizableClass):
 
         This method is absent in the original Python dict API.
         """
-        return [self[k] for k in self.newest_keys(max_n)]
+        return [self[k] for k in self.newest_keys(max_n)]