persidict 0.26.0__py3-none-any.whl → 0.30.0__py3-none-any.whl

persidict/__init__.py

@@ -24,8 +24,9 @@ and replace_unsafe_chars(), which replaces forbidden characters in a string.
 """
 from .safe_chars import get_safe_chars, replace_unsafe_chars
 from .safe_str_tuple import SafeStrTuple
-from .persi_dict import PersiDict
+from .persi_dict import PersiDict, PersiDictKey
 from .file_dir_dict import FileDirDict
 from .s3_dict import S3Dict
+from .write_once_dict import WriteOnceDict
 from .jokers import Joker, KeepCurrentFlag, DeleteCurrentFlag
 from .jokers import KEEP_CURRENT, DELETE_CURRENT

persidict/write_once_dict.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import time
+
+from deepdiff import DeepDiff
+from parameterizable import register_parameterizable_class
+from .jokers import KEEP_CURRENT, KeepCurrentFlag
+from .persi_dict import PersiDict
+from .file_dir_dict import FileDirDict
+import random
+import sys
+from typing import Any
+
+import joblib.hashing
+from .persi_dict import PersiDictKey
+
+def _get_md5_signature(x:Any) -> str:
+    """Return base16 MD5 hash signature of an object.
+
+    Uses joblib's Hasher (or NumpyHasher). It uses Pickle for serialization,
+    except for NumPy arrays, which use optimized custom routines.
+    """
+    if 'numpy' in sys.modules:
+        hasher = joblib.hashing.NumpyHasher(hash_name='md5')
+    else:
+        hasher = joblib.hashing.Hasher(hash_name='md5')
+    hash_signature = hasher.hash(x)
+    return str(hash_signature)
+
+class WriteOnceDict(PersiDict):
+    """ A dictionary that always keeps the first value assigned to a key.
+
+    If a key is already set, it randomly checks the value against the value
+    that was first set. If the new value is different, it raises a
+    ValueError exception. Once can control the frequency of these checks
+    or even completely disable them by setting `p_consistency_checks` to 0.
+
+    """
+    _wrapped_dict: PersiDict
+    _p_consistency_checks: float | None
+    _consistency_checks_attempted: int
+    _consistency_checks_passed: int
+
+    def __init__(self
+                 , wrapped_dict:PersiDict | None = None
+                 , p_consistency_checks: float | None=None):
+        if wrapped_dict is None:
+            wrapped_dict = FileDirDict(immutable_items = True)
+        assert isinstance(wrapped_dict, PersiDict)
+        assert wrapped_dict.immutable_items == True
+        self.p_consistency_checks = p_consistency_checks
+        PersiDict.__init__(self
+            , base_class_for_values=wrapped_dict.base_class_for_values
+            , immutable_items=True
+            , digest_len=wrapped_dict.digest_len)
+        self._wrapped_dict = wrapped_dict
+        self._consistency_checks_passed = 0
+        self._consistency_checks_attempted = 0
+
+
+    @property
+    def p_consistency_checks(self) -> float:
+        """ Probability of checking the value against the first value set. """
+        return self._p_consistency_checks
+
+
+    @p_consistency_checks.setter
+    def p_consistency_checks(self, value: float|None|KeepCurrentFlag) -> None:
+        if value is KEEP_CURRENT:
+            if hasattr(self, '_p_consistency_checks'):
+                return
+            else:
+                raise ValueError(
+                    f"KEEP_CURRENT can't be used to initialize p_consistency_checks.")
+        if value is None:
+            value = 0.0
+        if not (0 <= value <= 1):
+            raise ValueError(
+                f"p_consistency_checks must be in [0, 1], "
+                f"got {value}.")
+        self._p_consistency_checks = value
+
+
+    @property
+    def consistency_checks_failed(self) -> int:
+        """ Returns the number of failed consistency checks. """
+        return self._consistency_checks_attempted - self._consistency_checks_passed
+
+
+    @property
+    def consistency_checks_attempted(self) -> int:
+        """ Returns the number of attempted consistency checks. """
+        return self._consistency_checks_attempted
+
+
+    @property
+    def consistency_checks_passed(self) -> int:
+        """ Returns the number of successful consistency checks. """
+        return self._consistency_checks_passed
+
+
+    def get_params(self):
+        params = dict(
+            wrapped_dict = self._wrapped_dict,
+            p_consistency_checks = self.p_consistency_checks)
+        sorted_params = dict(sorted(params.items()))
+        return sorted_params
+
+    def __setitem__(self, key, value):
+        """ Set the value of a key if it is not already set.
+
+        If the key is already set, it checks the value
+        against the value that was first set.
+        """
+        check_needed = False
+
+        try: # extra protections to better handle concurrent writes
+            if key in self._wrapped_dict:
+                check_needed = True
+            else:
+                self._wrapped_dict[key] = value
+        except:
+            time.sleep(random.random()/random.randint(1,5))
+            if key in self._wrapped_dict:
+                check_needed = True
+            else:
+                self._wrapped_dict[key] = value
+
+        if not key in self._wrapped_dict:
+            raise KeyError(
+                f"Key {key} was not set in the wrapped dict "
+                + f"{self._wrapped_dict}. This should not happen.")
+
+        if check_needed and self.p_consistency_checks > 0:
+            if random.random() < self.p_consistency_checks:
+                self._consistency_checks_attempted += 1
+                signature_old = _get_md5_signature(self._wrapped_dict[key])
+                signature_new = _get_md5_signature(value)
+                if signature_old != signature_new:
+                    diff_dict = DeepDiff(self._wrapped_dict[key], value)
+                    raise ValueError(
+                        f"Key {key} is already set "
+                        + f"to {self._wrapped_dict[key]} "
+                        + f"and the new value {value} is different, "
+                        + f"which is not allowed. Details here: {diff_dict} ")
+                self._consistency_checks_passed += 1
+
+    def __contains__(self, item):
+        return item in self._wrapped_dict
+
+    def __getitem__(self, key):
+        return self._wrapped_dict[key]
+
+    def __len__(self):
+        return len(self._wrapped_dict)
+
+    def _generic_iter(self, iter_type: str):
+        return self._wrapped_dict._generic_iter(iter_type)
+
+    def timestamp(self, key:PersiDictKey) -> float:
+        return self._wrapped_dict.timestamp(key)
+
+    def __getattr__(self, name):
+        # Forward attribute access to the wrapped object
+        return getattr(self._wrapped_dict, name)
+
+    @property
+    def base_dir(self):
+        return self._wrapped_dict.base_dir
+
+    @property
+    def base_url(self):
+        return self._wrapped_dict.base_url
+
+    def get_subdict(self, prefix_key:PersiDictKey) -> WriteOnceDict:
+        subdict = self._wrapped_dict.get_subdict(prefix_key)
+        result = WriteOnceDict(subdict, self.p_consistency_checks)
+        return result
+
+register_parameterizable_class(WriteOnceDict)

{persidict-0.26.0.dist-info → persidict-0.30.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: persidict
-Version: 0.26.0
+Version: 0.30.0
 Summary: Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud.
 Keywords: persistence,dicts,distributed,parallel
 Author: Vlad (Volodymyr) Pavlov

{persidict-0.26.0.dist-info → persidict-0.30.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 persidict/.DS_Store,sha256=d65165279105ca6773180500688df4bdc69a2c7b771752f0a46ef120b7fd8ec3,6148
-persidict/__init__.py,sha256=25eacfa238e348bff6dfcf879e5f1f9de3b3509c233d47b67f2885f15c0ce5df,1266
+persidict/__init__.py,sha256=5280527b90ca7dc02affb44a878e3d036bdb4cd087b9db3ee2591d533a4ae2aa,1323
 persidict/file_dir_dict.py,sha256=4bb8c332f2fee27c95cdebde65406237e48f20e0783b474965e428ecf4b1bfae,14557
 persidict/jokers.py,sha256=917e1b13e8ca593336922ec93a69bfdae252f339bcbba22767f575db1a362262,1436
 persidict/persi_dict.py,sha256=7d62455e498e086f9df2ea795899c5f67370b587d580796de84837f15cd680e1,12207
@@ -7,6 +7,7 @@ persidict/s3_dict.py,sha256=140f6a2edd649f0394aa0a16f8438a00737fa6ebe38dbc856274
 persidict/safe_chars.py,sha256=59a20e96205d2e5675d827a911ad42ddbd553f1bd7e2cda1be765a9c2c4ce814,565
 persidict/safe_str_tuple.py,sha256=71393904bdebfb213ad8429fed59e04da52964076c01324f2238821aa4339325,3717
 persidict/safe_str_tuple_signing.py,sha256=e6e0a3015651a8ea2ef8aa43670f84c5ba1cdfedefb9ac8932aaebb7c699d1c9,3742
-persidict-0.26.0.dist-info/WHEEL,sha256=c133ef911c90b05f7e14d8679ba99146f9154fcd271b7398cf8f672283b94e05,79
-persidict-0.26.0.dist-info/METADATA,sha256=b43f5080ca1e0d4aeb00f8f0971069b99792eeb10414639ad2d3a56a837f28b7,9283
-persidict-0.26.0.dist-info/RECORD,,
+persidict/write_once_dict.py,sha256=58ca2d571de8c60a011e8f681d1d107689a24912ee2787d7761c5cee5cdedd3c,6379
+persidict-0.30.0.dist-info/WHEEL,sha256=c133ef911c90b05f7e14d8679ba99146f9154fcd271b7398cf8f672283b94e05,79
+persidict-0.30.0.dist-info/METADATA,sha256=0f2426f25aa4f4c76b08f66aaf7950edda7a652d0cf179dcbc5fc3ad58aaee04,9283
+persidict-0.30.0.dist-info/RECORD,,