persidict 0.25.0__tar.gz → 0.30.0__tar.gz

{persidict-0.25.0 → persidict-0.30.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: persidict
-Version: 0.25.0
+Version: 0.30.0
 Summary: Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud.
 Keywords: persistence,dicts,distributed,parallel
 Author: Vlad (Volodymyr) Pavlov
@@ -161,8 +161,9 @@ that simultaneously work with the same instance of a dictionary.
 `oldest_keys()`, `newest_values()`, `oldest_values()`, 
 `get_params()`, `get_metaparams()`, and `get_default_metaparams()`,
 which are not available in native Python dicts.
-* You can use NO_CHANGE constant as a fake new value 
-to avoid actually setting/updating a value.
+* You can use KEEP_CURRENT constant as a fake new value 
+to avoid actually setting/updating a value. Or DELETE_CURRENT as 
+a fake new value to delete the previous value from a dictionary.
 
 ## Fine Tuning
 

{persidict-0.25.0 → persidict-0.30.0}/README.md

@@ -128,8 +128,9 @@ that simultaneously work with the same instance of a dictionary.
 `oldest_keys()`, `newest_values()`, `oldest_values()`, 
 `get_params()`, `get_metaparams()`, and `get_default_metaparams()`,
 which are not available in native Python dicts.
-* You can use NO_CHANGE constant as a fake new value 
-to avoid actually setting/updating a value.
+* You can use KEEP_CURRENT constant as a fake new value 
+to avoid actually setting/updating a value. Or DELETE_CURRENT as 
+a fake new value to delete the previous value from a dictionary.
 
 ## Fine Tuning
 

{persidict-0.25.0 → persidict-0.30.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "persidict"
-version = "0.25.0"
+version = "0.30.0"
 description = "Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud."
 readme = "README.md"
 requires-python = ">=3.10"

{persidict-0.25.0 → persidict-0.30.0}/src/persidict/__init__.py

@@ -24,7 +24,9 @@ and replace_unsafe_chars(), which replaces forbidden characters in a string.
 """
 from .safe_chars import get_safe_chars, replace_unsafe_chars
 from .safe_str_tuple import SafeStrTuple
-from .persi_dict import PersiDict
+from .persi_dict import PersiDict, PersiDictKey
 from .file_dir_dict import FileDirDict
 from .s3_dict import S3Dict
-from .nochange_const import *
+from .write_once_dict import WriteOnceDict
+from .jokers import Joker, KeepCurrentFlag, DeleteCurrentFlag
+from .jokers import KEEP_CURRENT, DELETE_CURRENT

{persidict-0.25.0 → persidict-0.30.0}/src/persidict/file_dir_dict.py

@@ -12,7 +12,6 @@ from __future__ import annotations
 import os
 import random
 import time
-from abc import abstractmethod
 from typing import Any, Optional
 
 import joblib
@@ -21,7 +20,7 @@ import jsonpickle.ext.numpy as jsonpickle_numpy
 import jsonpickle.ext.pandas as jsonpickle_pandas
 import parameterizable
 
-from .nochange_const import NO_CHANGE
+from .jokers import KEEP_CURRENT, DELETE_CURRENT, Joker
 from .safe_chars import replace_unsafe_chars
 from .safe_str_tuple import SafeStrTuple
 from .safe_str_tuple_signing import sign_safe_str_tuple, unsign_safe_str_tuple
@@ -317,7 +316,11 @@ class FileDirDict(PersiDict):
     def __setitem__(self, key:PersiDictKey, value:Any):
         """Set self[key] to value."""
 
-        if value is NO_CHANGE:
+        if value is KEEP_CURRENT:
+            return
+
+        if value is DELETE_CURRENT:
+            self.delete_if_exists(key)
             return
 
         if isinstance(value, PersiDict):

persidict-0.30.0/src/persidict/jokers.py

@@ -0,0 +1,51 @@
+"""A singleton constant to indicate no change in a value.
+
+When updating a value in a persistent dictionary,
+use KEEP_CURRENT as the new value to indicate that
+the existing value should remain unchanged.
+"""
+from typing import Any
+
+from parameterizable import (
+    ParameterizableClass
+    , register_parameterizable_class)
+
+
+class Joker(ParameterizableClass):
+    _instances = {}
+
+    def get_params(self) -> dict[str, Any]:
+        return {}
+
+    def __new__(cls):
+        if cls not in Joker._instances:
+            Joker._instances[cls] = super().__new__(cls)
+        return Joker._instances[cls]
+
+
+class KeepCurrentFlag(Joker):
+    """A singleton constant to indicate no change in a value.
+
+    When updating a value in a persistent dictionary,
+    use KeepCurrent as the new value to indicate that
+    the existing value (if any) should remain unchanged.
+    """
+    pass
+
+class DeleteCurrentFlag(Joker):
+    """A singleton constant to indicate that the current value should be deleted.
+
+    When updating a value in a persistent dictionary,
+    use DeleteCurrentFlag as the new value to indicate that
+    the existing value (if any) should be removed from the dictionary.
+    """
+    pass
+
+register_parameterizable_class(KeepCurrentFlag)
+register_parameterizable_class(DeleteCurrentFlag)
+
+KeepCurrent = KeepCurrentFlag()
+KEEP_CURRENT = KeepCurrentFlag()
+
+DeleteCurrent = DeleteCurrentFlag()
+DELETE_CURRENT = DeleteCurrentFlag()

{persidict-0.25.0 → persidict-0.30.0}/src/persidict/persi_dict.py

@@ -26,7 +26,7 @@ from parameterizable import ParameterizableClass
 from typing import Any, Sequence, Optional
 from collections.abc import MutableMapping
 
-from .nochange_const import NO_CHANGE
+from .jokers import KEEP_CURRENT, DELETE_CURRENT, Joker
 from .safe_str_tuple import SafeStrTuple
 
 PersiDictKey = SafeStrTuple | Sequence[str] | str
@@ -161,9 +161,11 @@ class PersiDict(MutableMapping, ParameterizableClass):
 
     def __setitem__(self, key:PersiDictKey, value:Any):
         """Set self[key] to value."""
-        if value is NO_CHANGE:
+        if value is KEEP_CURRENT:
             return
-        if self.immutable_items:
+        elif value is DELETE_CURRENT:
+            self.delete_if_exists(key)
+        elif self.immutable_items:
             if key in self:
                 raise KeyError("Can't modify an immutable key-value pair")
         raise NotImplementedError
@@ -216,7 +218,7 @@ class PersiDict(MutableMapping, ParameterizableClass):
         """
         # TODO: check edge cases to ensure the same semantics as standard dicts
         key = SafeStrTuple(key)
-        assert not default is NO_CHANGE
+        assert not isinstance(default, Joker)
         if key in self:
             return self[key]
         else:

{persidict-0.25.0 → persidict-0.30.0}/src/persidict/s3_dict.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import os
-from abc import abstractmethod
 from typing import Any, Optional
 
 import boto3
@@ -10,7 +9,7 @@ import parameterizable
 from .safe_str_tuple import SafeStrTuple
 from .safe_str_tuple_signing import sign_safe_str_tuple, unsign_safe_str_tuple
 from .persi_dict import PersiDict
-from .nochange_const import NO_CHANGE
+from .jokers import KEEP_CURRENT, DELETE_CURRENT
 from .file_dir_dict import FileDirDict, PersiDictKey
 
 S3DICT_DEFAULT_BASE_DIR = "__s3_dict__"
@@ -191,7 +190,11 @@ class S3Dict(PersiDict):
     def __setitem__(self, key:PersiDictKey, value:Any):
         """Set self[key] to value. """
 
-        if value is NO_CHANGE:
+        if value is KEEP_CURRENT:
+            return
+
+        if value is DELETE_CURRENT:
+            self.delete_if_exists(key)
             return
 
         if isinstance(value, PersiDict):

persidict-0.30.0/src/persidict/write_once_dict.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import time
+
+from deepdiff import DeepDiff
+from parameterizable import register_parameterizable_class
+from .jokers import KEEP_CURRENT, KeepCurrentFlag
+from .persi_dict import PersiDict
+from .file_dir_dict import FileDirDict
+import random
+import sys
+from typing import Any
+
+import joblib.hashing
+from .persi_dict import PersiDictKey
+
+def _get_md5_signature(x:Any) -> str:
+    """Return base16 MD5 hash signature of an object.
+
+    Uses joblib's Hasher (or NumpyHasher). It uses Pickle for serialization,
+    except for NumPy arrays, which use optimized custom routines.
+    """
+    if 'numpy' in sys.modules:
+        hasher = joblib.hashing.NumpyHasher(hash_name='md5')
+    else:
+        hasher = joblib.hashing.Hasher(hash_name='md5')
+    hash_signature = hasher.hash(x)
+    return str(hash_signature)
+
+class WriteOnceDict(PersiDict):
+    """ A dictionary that always keeps the first value assigned to a key.
+
+    If a key is already set, it randomly checks the value against the value
+    that was first set. If the new value is different, it raises a
+    ValueError exception. Once can control the frequency of these checks
+    or even completely disable them by setting `p_consistency_checks` to 0.
+
+    """
+    _wrapped_dict: PersiDict
+    _p_consistency_checks: float | None
+    _consistency_checks_attempted: int
+    _consistency_checks_passed: int
+
+    def __init__(self
+                 , wrapped_dict:PersiDict | None = None
+                 , p_consistency_checks: float | None=None):
+        if wrapped_dict is None:
+            wrapped_dict = FileDirDict(immutable_items = True)
+        assert isinstance(wrapped_dict, PersiDict)
+        assert wrapped_dict.immutable_items == True
+        self.p_consistency_checks = p_consistency_checks
+        PersiDict.__init__(self
+            , base_class_for_values=wrapped_dict.base_class_for_values
+            , immutable_items=True
+            , digest_len=wrapped_dict.digest_len)
+        self._wrapped_dict = wrapped_dict
+        self._consistency_checks_passed = 0
+        self._consistency_checks_attempted = 0
+
+
+    @property
+    def p_consistency_checks(self) -> float:
+        """ Probability of checking the value against the first value set. """
+        return self._p_consistency_checks
+
+
+    @p_consistency_checks.setter
+    def p_consistency_checks(self, value: float|None|KeepCurrentFlag) -> None:
+        if value is KEEP_CURRENT:
+            if hasattr(self, '_p_consistency_checks'):
+                return
+            else:
+                raise ValueError(
+                    f"KEEP_CURRENT can't be used to initialize p_consistency_checks.")
+        if value is None:
+            value = 0.0
+        if not (0 <= value <= 1):
+            raise ValueError(
+                f"p_consistency_checks must be in [0, 1], "
+                f"got {value}.")
+        self._p_consistency_checks = value
+
+
+    @property
+    def consistency_checks_failed(self) -> int:
+        """ Returns the number of failed consistency checks. """
+        return self._consistency_checks_attempted - self._consistency_checks_passed
+
+
+    @property
+    def consistency_checks_attempted(self) -> int:
+        """ Returns the number of attempted consistency checks. """
+        return self._consistency_checks_attempted
+
+
+    @property
+    def consistency_checks_passed(self) -> int:
+        """ Returns the number of successful consistency checks. """
+        return self._consistency_checks_passed
+
+
+    def get_params(self):
+        params = dict(
+            wrapped_dict = self._wrapped_dict,
+            p_consistency_checks = self.p_consistency_checks)
+        sorted_params = dict(sorted(params.items()))
+        return sorted_params
+
+    def __setitem__(self, key, value):
+        """ Set the value of a key if it is not already set.
+
+        If the key is already set, it checks the value
+        against the value that was first set.
+        """
+        check_needed = False
+
+        try: # extra protections to better handle concurrent writes
+            if key in self._wrapped_dict:
+                check_needed = True
+            else:
+                self._wrapped_dict[key] = value
+        except:
+            time.sleep(random.random()/random.randint(1,5))
+            if key in self._wrapped_dict:
+                check_needed = True
+            else:
+                self._wrapped_dict[key] = value
+
+        if not key in self._wrapped_dict:
+            raise KeyError(
+                f"Key {key} was not set in the wrapped dict "
+                + f"{self._wrapped_dict}. This should not happen.")
+
+        if check_needed and self.p_consistency_checks > 0:
+            if random.random() < self.p_consistency_checks:
+                self._consistency_checks_attempted += 1
+                signature_old = _get_md5_signature(self._wrapped_dict[key])
+                signature_new = _get_md5_signature(value)
+                if signature_old != signature_new:
+                    diff_dict = DeepDiff(self._wrapped_dict[key], value)
+                    raise ValueError(
+                        f"Key {key} is already set "
+                        + f"to {self._wrapped_dict[key]} "
+                        + f"and the new value {value} is different, "
+                        + f"which is not allowed. Details here: {diff_dict} ")
+                self._consistency_checks_passed += 1
+
+    def __contains__(self, item):
+        return item in self._wrapped_dict
+
+    def __getitem__(self, key):
+        return self._wrapped_dict[key]
+
+    def __len__(self):
+        return len(self._wrapped_dict)
+
+    def _generic_iter(self, iter_type: str):
+        return self._wrapped_dict._generic_iter(iter_type)
+
+    def timestamp(self, key:PersiDictKey) -> float:
+        return self._wrapped_dict.timestamp(key)
+
+    def __getattr__(self, name):
+        # Forward attribute access to the wrapped object
+        return getattr(self._wrapped_dict, name)
+
+    @property
+    def base_dir(self):
+        return self._wrapped_dict.base_dir
+
+    @property
+    def base_url(self):
+        return self._wrapped_dict.base_url
+
+    def get_subdict(self, prefix_key:PersiDictKey) -> WriteOnceDict:
+        subdict = self._wrapped_dict.get_subdict(prefix_key)
+        result = WriteOnceDict(subdict, self.p_consistency_checks)
+        return result
+
+register_parameterizable_class(WriteOnceDict)

persidict-0.25.0/src/persidict/nochange_const.py

@@ -1,28 +0,0 @@
-"""A singleton constant to indicate no change in a value.
-
-When updating a value in a persistent dictionary,
-use NO_CHANGE as the new value to indicate that
-the existing value should remain unchanged.
-"""
-from typing import Any
-
-from parameterizable import (
-    ParameterizableClass
-    , register_parameterizable_class)
-
-
-class NoChangeFlag(ParameterizableClass):
-    _instance = None
-
-    def __new__(cls):
-        if cls._instance is None:
-            cls._instance = super().__new__(cls)
-        return cls._instance
-
-    def get_params(self) -> dict[str, Any]:
-        return {}
-
-register_parameterizable_class(NoChangeFlag)
-
-NoChange = NoChangeFlag()
-NO_CHANGE = NoChangeFlag()