persidict 0.31.0__tar.gz → 0.32.0__tar.gz

{persidict-0.31.0 → persidict-0.32.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: persidict
-Version: 0.31.0
+Version: 0.32.0
 Summary: Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud.
 Keywords: persistence,dicts,distributed,parallel
 Author: Vlad (Volodymyr) Pavlov
@@ -20,7 +20,8 @@ Requires-Dist: joblib
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: jsonpickle
-Requires-Dist: parameterizable
+Requires-Dist: joblib
+Requires-Dist: deepdiff
 Requires-Dist: boto3 ; extra == 'aws'
 Requires-Dist: boto3 ; extra == 'dev'
 Requires-Dist: moto ; extra == 'dev'
@@ -37,7 +38,7 @@ Simple persistent dictionaries for Python.
 
 ## What Is It?
 
-`persidict` offers a simple persistent key-value store for Python. 
+`persidict` offers a very simple persistent key-value store for Python. 
 It saves the content of the dictionary in a folder on a disk 
 or in an S3 bucket on AWS. Each value is stored as a separate file / S3 object.
 Only text strings or sequences of strings are allowed as keys.

{persidict-0.31.0 → persidict-0.32.0}/README.md

@@ -4,7 +4,7 @@ Simple persistent dictionaries for Python.
 
 ## What Is It?
 
-`persidict` offers a simple persistent key-value store for Python. 
+`persidict` offers a very simple persistent key-value store for Python. 
 It saves the content of the dictionary in a folder on a disk 
 or in an S3 bucket on AWS. Each value is stored as a separate file / S3 object.
 Only text strings or sequences of strings are allowed as keys.

{persidict-0.31.0 → persidict-0.32.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "persidict"
-version = "0.31.0"
+version = "0.32.0"
 description = "Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -30,7 +30,8 @@ dependencies = [
     "numpy",
     "pandas",
     "jsonpickle",
-    "parameterizable"
+    "joblib",
+    "deepdiff"
 ]
 
 [project.urls]
@@ -45,4 +46,4 @@ dev = [
 
 aws = [
     "boto3"
-]
+]

{persidict-0.31.0 → persidict-0.32.0}/src/persidict/file_dir_dict.py

@@ -19,6 +19,7 @@ import jsonpickle
 import jsonpickle.ext.numpy as jsonpickle_numpy
 import jsonpickle.ext.pandas as jsonpickle_pandas
 import parameterizable
+from parameterizable import sort_dict_by_keys
 
 from .jokers import KEEP_CURRENT, DELETE_CURRENT, Joker
 from .safe_chars import replace_unsafe_chars
@@ -124,7 +125,7 @@ class FileDirDict(PersiDict):
             base_dir=self.base_dir
             , file_type=self.file_type)
         params.update(additional_params)
-        sorted_params = dict(sorted(params.items()))
+        sorted_params = sort_dict_by_keys(params)
         return sorted_params
 
 
@@ -151,11 +152,20 @@ class FileDirDict(PersiDict):
 
         num_files = 0
         suffix = "." + self.file_type
-        for subdir_info in os.walk(self._base_dir):
-            files = subdir_info[2]
-            files = [f_name for f_name in files
-                     if f_name.endswith(suffix)]
-            num_files += len(files)
+        stack = [self._base_dir]
+
+        while stack:
+            path = stack.pop()
+            try:
+                with os.scandir(path) as it:
+                    for entry in it:
+                        if entry.is_dir(follow_symlinks=False):
+                            stack.append(entry.path)
+                        elif entry.is_file(follow_symlinks=False) and entry.name.endswith(suffix):
+                            num_files += 1
+            except PermissionError:
+                continue
+
         return num_files
 
 
@@ -202,7 +212,39 @@ class FileDirDict(PersiDict):
             file_name = key[-1] + "." + self.file_type
             return os.path.join(*dir_names, file_name)
         else:
-            return os.path.join(*dir_names)
+            return str(os.path.join(*dir_names))
+
+
+    def _build_key_from_full_path(self, full_path:str)->SafeStrTuple:
+        """Convert a filesystem path back into a key."""
+
+        # Ensure we're working with absolute paths
+        full_path = os.path.abspath(full_path)
+
+        # Remove the base directory from the path
+        if not full_path.startswith(self._base_dir):
+            raise ValueError(f"Path {full_path} is not within base directory {self._base_dir}")
+
+        # Get the relative path
+        rel_path = full_path[len(self._base_dir):].lstrip(os.sep)
+
+        if not rel_path:
+            return SafeStrTuple()
+
+        # Split the path into components
+        path_components = rel_path.split(os.sep)
+
+        # If it's a file path, remove the file extension from the last component
+        if os.path.isfile(full_path) and path_components[-1].endswith("." + self.file_type):
+            path_components[-1] = path_components[-1][:-len("." + self.file_type)]
+
+        # Create a SafeStrTuple from the path components
+        key = SafeStrTuple(*path_components)
+
+        # Unsign the key
+        key = unsign_safe_str_tuple(key, self.digest_len)
+
+        return key
 
 
     def get_subdict(self, key:PersiDictKey) -> FileDirDict:
@@ -403,4 +445,46 @@ class FileDirDict(PersiDict):
         return os.path.getmtime(filename)
 
 
-parameterizable.register_parameterizable_class(FileDirDict)
+    def random_key(self) -> PersiDictKey | None:
+        # canonicalise extension once
+        early_exit_cap = 10_000
+        ext = None
+        if self.file_type:
+            ext = self.file_type.lower()
+            if not ext.startswith("."):
+                ext = "." + ext
+
+        stack = [self._base_dir]
+        winner: Optional[str] = None
+        seen = 0
+
+        while stack:
+            path = stack.pop()
+            try:
+                with os.scandir(path) as it:
+                    for ent in it:
+                        if ent.is_dir(follow_symlinks=False):
+                            stack.append(ent.path)
+                            continue
+
+                        # cheap name test before stat()
+                        if ext and not ent.name.lower().endswith(ext):
+                            continue
+
+                        if ent.is_file(follow_symlinks=False):
+                            seen += 1
+                            if random.random() < 1 / seen:  # reservoir k=1
+                                winner = ent.path
+                            # early‑exit when cap reached
+                            if early_exit_cap and seen >= early_exit_cap:
+                                return self._build_key_from_full_path(os.path.abspath(winner))
+            except PermissionError:
+                continue
+
+        if winner is None:
+            return None
+        else:
+            return self._build_key_from_full_path(os.path.abspath(winner))
+
+
+parameterizable.register_parameterizable_class(FileDirDict)

{persidict-0.31.0 → persidict-0.32.0}/src/persidict/persi_dict.py

@@ -22,7 +22,7 @@ from __future__ import annotations
 
 from abc import abstractmethod
 import random
-from parameterizable import ParameterizableClass
+from parameterizable import ParameterizableClass, sort_dict_by_keys
 from typing import Any, Sequence, Optional
 from collections.abc import MutableMapping
 
@@ -107,7 +107,7 @@ class PersiDict(MutableMapping, ParameterizableClass):
             , digest_len=self.digest_len
             , base_class_for_values=self.base_class_for_values
         )
-        sorted_params = dict(sorted(params.items()))
+        sorted_params = sort_dict_by_keys(params)
         return sorted_params
 
 
@@ -302,15 +302,33 @@ class PersiDict(MutableMapping, ParameterizableClass):
         return result_subdicts
 
 
-    def random_keys(self, max_n:int):
-        """Return a list of random keys from the dictionary.
+    def random_key(self) -> PersiDictKey | None:
+        """Return a random key from the dictionary.
+
+        Returns a single random key if the dictionary is not empty.
+        Returns None if the dictionary is empty.
 
         This method is absent in the original Python dict API.
+
+        Implementation uses reservoir sampling to select a uniformly random key
+        in streaming time, without loading all keys into memory or using len().
         """
-        all_keys = list(self.keys())
-        if max_n > len(all_keys):
-            max_n = len(all_keys)
-        result = random.sample(all_keys, max_n)
+        iterator = iter(self.keys())
+        try:
+            # Get the first key
+            result = next(iterator)
+        except StopIteration:
+            # Dictionary is empty
+            return None
+
+        # Reservoir sampling algorithm
+        i = 2
+        for key in iterator:
+            # Select current key with probability 1/i
+            if random.random() < 1/i:
+                result = key
+            i += 1
+
         return result
 
 
@@ -372,4 +390,4 @@ class PersiDict(MutableMapping, ParameterizableClass):
 
         This method is absent in the original Python dict API.
         """
-        return [self[k] for k in self.newest_keys(max_n)]
+        return [self[k] for k in self.newest_keys(max_n)]

{persidict-0.31.0 → persidict-0.32.0}/src/persidict/s3_dict.py

@@ -5,6 +5,7 @@ from typing import Any, Optional
 
 import boto3
 import parameterizable
+from parameterizable.dict_sorter import sort_dict_by_keys
 
 from .safe_str_tuple import SafeStrTuple
 from .safe_str_tuple_signing import sign_safe_str_tuple, unsign_safe_str_tuple
@@ -119,7 +120,7 @@ class S3Dict(PersiDict):
         params["region"] = self.region
         params["bucket_name"] = self.bucket_name
         params["root_prefix"] = self.root_prefix
-        sorted_params = dict(sorted(params.items()))
+        sorted_params = sort_dict_by_keys(params)
         return sorted_params
 
 

{persidict-0.31.0 → persidict-0.32.0}/src/persidict/write_once_dict.py

@@ -3,7 +3,8 @@ from __future__ import annotations
 import time
 
 from deepdiff import DeepDiff
-from parameterizable import register_parameterizable_class
+from parameterizable import register_parameterizable_class, sort_dict_by_keys
+
 from .jokers import KEEP_CURRENT, KeepCurrentFlag
 from .persi_dict import PersiDict
 from .file_dir_dict import FileDirDict
@@ -103,7 +104,7 @@ class WriteOnceDict(PersiDict):
         params = dict(
             wrapped_dict = self._wrapped_dict,
             p_consistency_checks = self.p_consistency_checks)
-        sorted_params = dict(sorted(params.items()))
+        sorted_params = sort_dict_by_keys(params)
         return sorted_params
 
     def __setitem__(self, key, value):