persidict 0.105.4__tar.gz → 0.105.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of persidict might be problematic. Click here for more details.
- {persidict-0.105.4 → persidict-0.105.5}/PKG-INFO +1 -1
- {persidict-0.105.4 → persidict-0.105.5}/pyproject.toml +1 -1
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/__init__.py +7 -0
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/basic_s3_dict.py +13 -4
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/cached_appendonly_dict.py +53 -28
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/file_dir_dict.py +0 -2
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/local_dict.py +0 -3
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/s3_dict_file_dir_cached.py +1 -4
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/singletons.py +0 -8
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/write_once_dict.py +2 -12
- {persidict-0.105.4 → persidict-0.105.5}/README.md +0 -0
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/.DS_Store +0 -0
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/cached_mutable_dict.py +0 -0
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/empty_dict.py +0 -0
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/overlapping_multi_dict.py +0 -0
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/persi_dict.py +0 -0
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/safe_chars.py +0 -0
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/safe_str_tuple.py +0 -0
- {persidict-0.105.4 → persidict-0.105.5}/src/persidict/safe_str_tuple_signing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: persidict
|
|
3
|
-
Version: 0.105.
|
|
3
|
+
Version: 0.105.5
|
|
4
4
|
Summary: Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud.
|
|
5
5
|
Keywords: persistence,dicts,distributed,parallel
|
|
6
6
|
Author: Vlad (Volodymyr) Pavlov
|
|
@@ -4,7 +4,7 @@ build-backend = "uv_build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "persidict"
|
|
7
|
-
version = "0.105.
|
|
7
|
+
version = "0.105.5"
|
|
8
8
|
description = "Simple persistent key-value store for Python. Values are stored as files on a disk or as S3 objects on AWS cloud."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -4,6 +4,7 @@ This package provides a unified interface for persistent dictionary-like
|
|
|
4
4
|
storage with various backends including filesystem and AWS S3.
|
|
5
5
|
|
|
6
6
|
Classes:
|
|
7
|
+
|
|
7
8
|
PersiDict: Abstract base class defining the unified interface for all
|
|
8
9
|
persistent dictionaries.
|
|
9
10
|
NonEmptySafeStrTuple: A flat tuple of URL/filename-safe strings that
|
|
@@ -11,24 +12,30 @@ Classes:
|
|
|
11
12
|
FileDirDict: A dictionary that stores key-value pairs as files on a
|
|
12
13
|
local hard drive. Keys compose filenames, values are stored as
|
|
13
14
|
pickle or JSON objects.
|
|
15
|
+
|
|
14
16
|
BasicS3Dict: A basic S3-backed dictionary with direct S3 operations.
|
|
17
|
+
|
|
15
18
|
WriteOnceDict: A write-once wrapper that prevents modification of existing
|
|
16
19
|
items after initial storage.
|
|
17
20
|
EmptyDict: Equivalent of null device in OS - accepts all writes but discards
|
|
18
21
|
them, returns nothing on reads. Always appears empty regardless of
|
|
19
22
|
operations performed. Useful for testing, debugging, or as a placeholder.
|
|
23
|
+
|
|
20
24
|
OverlappingMultiDict: A dictionary that can handle overlapping key spaces.
|
|
21
25
|
|
|
22
26
|
Functions:
|
|
27
|
+
|
|
23
28
|
get_safe_chars(): Returns a set of URL/filename-safe characters permitted
|
|
24
29
|
in keys.
|
|
25
30
|
replace_unsafe_chars(): Replaces forbidden characters in a string with
|
|
26
31
|
safe alternatives.
|
|
27
32
|
|
|
28
33
|
Constants:
|
|
34
|
+
|
|
29
35
|
KEEP_CURRENT, DELETE_CURRENT: Special joker values for conditional operations.
|
|
30
36
|
|
|
31
37
|
Note:
|
|
38
|
+
|
|
32
39
|
All persistent dictionaries support multiple serialization formats, including
|
|
33
40
|
pickle and JSON, with automatic type handling and collision-safe key encoding.
|
|
34
41
|
"""
|
|
@@ -165,7 +165,18 @@ class BasicS3Dict(PersiDict):
|
|
|
165
165
|
|
|
166
166
|
|
|
167
167
|
def etag(self, key:NonEmptyPersiDictKey) -> str|None:
|
|
168
|
-
"""Get an ETag for a key.
|
|
168
|
+
"""Get an ETag for a key.
|
|
169
|
+
|
|
170
|
+
Args:
|
|
171
|
+
key: Dictionary key (string or sequence of strings
|
|
172
|
+
or NonEmptySafeStrTuple).
|
|
173
|
+
|
|
174
|
+
Returns:
|
|
175
|
+
str|None: The ETag value for the S3 object, or None if not available.
|
|
176
|
+
|
|
177
|
+
Raises:
|
|
178
|
+
KeyError: If the key does not exist in S3.
|
|
179
|
+
"""
|
|
169
180
|
key = NonEmptySafeStrTuple(key)
|
|
170
181
|
obj_name = self._build_full_objectname(key)
|
|
171
182
|
try:
|
|
@@ -590,6 +601,4 @@ class BasicS3Dict(PersiDict):
|
|
|
590
601
|
if not_found_error(e):
|
|
591
602
|
raise KeyError(f"Key {key} not found in S3 bucket {self.bucket_name}")
|
|
592
603
|
else:
|
|
593
|
-
raise
|
|
594
|
-
|
|
595
|
-
# parameterizable.register_parameterizable_class(BasicS3Dict)
|
|
604
|
+
raise
|
|
@@ -1,3 +1,23 @@
|
|
|
1
|
+
"""Read-through cached, append-only persistent dictionary adapter.
|
|
2
|
+
|
|
3
|
+
This module provides `AppendOnlyDictCached`, an append-only facade that
|
|
4
|
+
combines two concrete `PersiDict` implementations:
|
|
5
|
+
|
|
6
|
+
- `main_dict`: the authoritative store (source of truth) where data is
|
|
7
|
+
actually persisted;
|
|
8
|
+
- `data_cache`: a secondary `PersiDict` that is used strictly as a cache for
|
|
9
|
+
values.
|
|
10
|
+
|
|
11
|
+
Because both backends are append-only (items may be added once and never
|
|
12
|
+
modified or deleted), the cache can be trusted once it has a value for a key.
|
|
13
|
+
Reads go to the cache first and fall back to the main dict on a miss, at which
|
|
14
|
+
point the cache is populated. Writes always go to the main dict first and are
|
|
15
|
+
mirrored to the cache after validation performed by the `PersiDict` base.
|
|
16
|
+
|
|
17
|
+
The adapter delegates iteration, length, timestamps, and base properties to the
|
|
18
|
+
main dict to keep semantics consistent with the authoritative store.
|
|
19
|
+
"""
|
|
20
|
+
|
|
1
21
|
from __future__ import annotations
|
|
2
22
|
|
|
3
23
|
from typing import Any, Optional
|
|
@@ -8,40 +28,41 @@ from .singletons import ETAG_HAS_NOT_CHANGED, EXECUTION_IS_COMPLETE
|
|
|
8
28
|
|
|
9
29
|
|
|
10
30
|
class AppendOnlyDictCached(PersiDict):
|
|
11
|
-
"""Append-only
|
|
12
|
-
|
|
13
|
-
This adapter wraps two concrete PersiDict instances:
|
|
14
|
-
- main_dict: the source of truth that actually persists data.
|
|
15
|
-
- data_cache: a second PersiDict used purely as a cache for values.
|
|
31
|
+
"""Append-only `PersiDict` facade with a read-through cache.
|
|
16
32
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
33
|
+
This adapter composes two concrete `PersiDict` instances and presents them
|
|
34
|
+
as a single append-only mapping. It trusts the cache because both backends
|
|
35
|
+
are append-only: once a key is written it will never be modified or
|
|
36
|
+
deleted.
|
|
21
37
|
|
|
22
38
|
Behavior summary:
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
-
|
|
33
|
-
|
|
34
|
-
|
|
39
|
+
|
|
40
|
+
- Reads: `__getitem__` first tries the cache, falls back to the main dict,
|
|
41
|
+
then populates the cache on a miss.
|
|
42
|
+
- Membership: `__contains__` returns True immediately if the key is in the
|
|
43
|
+
cache; otherwise it checks the main dict.
|
|
44
|
+
- Writes: `__setitem__` writes to the main dict and then mirrors the value
|
|
45
|
+
into the cache (after base validation performed by `PersiDict`).
|
|
46
|
+
- `set_item_get_etag`: delegates the write to the main dict, mirrors the
|
|
47
|
+
value into the cache, and returns the ETag from the main dict.
|
|
48
|
+
- Deletion: not supported (append-only), will raise `TypeError`.
|
|
49
|
+
- Iteration/length/timestamps: delegated to the main dict.
|
|
50
|
+
|
|
51
|
+
Attributes:
|
|
52
|
+
_main: The authoritative append-only `PersiDict` instance.
|
|
53
|
+
_data_cache: The append-only `PersiDict` used purely as a value cache.
|
|
35
54
|
|
|
36
55
|
Args:
|
|
37
|
-
main_dict: The authoritative append-only PersiDict
|
|
38
|
-
data_cache: A PersiDict used as a
|
|
39
|
-
compatible with main_dict
|
|
56
|
+
main_dict: The authoritative append-only `PersiDict`.
|
|
57
|
+
data_cache: A `PersiDict` used as a cache; must be append-only and
|
|
58
|
+
compatible with `main_dict` (same `base_class_for_values` and
|
|
59
|
+
`serialization_format`).
|
|
40
60
|
|
|
41
61
|
Raises:
|
|
42
|
-
TypeError: If main_dict or data_cache are not PersiDict instances.
|
|
43
|
-
ValueError: If either dict is not
|
|
44
|
-
base_class_for_values differ.
|
|
62
|
+
TypeError: If `main_dict` or `data_cache` are not `PersiDict` instances.
|
|
63
|
+
ValueError: If either dict is not append-only or their
|
|
64
|
+
`base_class_for_values` differ.
|
|
65
|
+
|
|
45
66
|
"""
|
|
46
67
|
|
|
47
68
|
def __init__(self,
|
|
@@ -101,7 +122,11 @@ class AppendOnlyDictCached(PersiDict):
|
|
|
101
122
|
return key in self._main
|
|
102
123
|
|
|
103
124
|
def __len__(self) -> int:
|
|
104
|
-
"""
|
|
125
|
+
"""Return the number of items.
|
|
126
|
+
|
|
127
|
+
Returns:
|
|
128
|
+
int: Number of items in the dictionary, delegated to the main dict.
|
|
129
|
+
"""
|
|
105
130
|
return len(self._main)
|
|
106
131
|
|
|
107
132
|
def _generic_iter(self, result_type: set[str]):
|
|
@@ -209,7 +209,4 @@ class S3Dict_FileDirCached(PersiDict):
|
|
|
209
209
|
return False
|
|
210
210
|
|
|
211
211
|
|
|
212
|
-
S3Dict = S3Dict_FileDirCached # Alias for backward compatibility
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
# parameterizable.register_parameterizable_class(S3Dict_FileDirCached)
|
|
212
|
+
S3Dict = S3Dict_FileDirCached # Alias for backward compatibility
|
|
@@ -18,7 +18,6 @@ Examples:
|
|
|
18
18
|
from typing import Any
|
|
19
19
|
|
|
20
20
|
from parameterizable import ParameterizableClass
|
|
21
|
-
# , register_parameterizable_class)
|
|
22
21
|
|
|
23
22
|
|
|
24
23
|
class Singleton(ParameterizableClass):
|
|
@@ -140,13 +139,6 @@ class ExecutionIsCompleteFlag(StatusFlag):
|
|
|
140
139
|
"""
|
|
141
140
|
pass
|
|
142
141
|
|
|
143
|
-
|
|
144
|
-
# register_parameterizable_class(KeepCurrentFlag)
|
|
145
|
-
# register_parameterizable_class(DeleteCurrentFlag)
|
|
146
|
-
# register_parameterizable_class(ContinueNormalExecutionFlag)
|
|
147
|
-
# register_parameterizable_class(ExecutionIsCompleteFlag)
|
|
148
|
-
# register_parameterizable_class(ETagHasNotChangedFlag)
|
|
149
|
-
|
|
150
142
|
_KeepCurrent = KeepCurrentFlag()
|
|
151
143
|
KEEP_CURRENT = KeepCurrentFlag()
|
|
152
144
|
"""Flag indicating that the current value should be kept unchanged.
|
|
@@ -13,7 +13,7 @@ from __future__ import annotations
|
|
|
13
13
|
import time
|
|
14
14
|
|
|
15
15
|
from deepdiff import DeepDiff
|
|
16
|
-
from parameterizable import sort_dict_by_keys
|
|
16
|
+
from parameterizable import sort_dict_by_keys
|
|
17
17
|
|
|
18
18
|
from .singletons import KEEP_CURRENT, KeepCurrentFlag, ETagHasNotChangedFlag
|
|
19
19
|
from .persi_dict import PersiDict, NonEmptyPersiDictKey
|
|
@@ -61,13 +61,6 @@ class WriteOnceDict(PersiDict):
|
|
|
61
61
|
and you want to check this assumption (detect divergent values)
|
|
62
62
|
without paying the full cost of always comparing values.
|
|
63
63
|
|
|
64
|
-
Attributes:
|
|
65
|
-
p_consistency_checks (float): Probability in [0, 1] of performing a
|
|
66
|
-
consistency check for a key that has been previously set.
|
|
67
|
-
consistency_checks_attempted (int): Number of checks that were
|
|
68
|
-
attempted.
|
|
69
|
-
consistency_checks_passed (int): Number of checks that succeeded.
|
|
70
|
-
consistency_checks_failed (int): Derived as attempted - passed.
|
|
71
64
|
|
|
72
65
|
"""
|
|
73
66
|
_wrapped_dict: PersiDict
|
|
@@ -343,7 +336,4 @@ class WriteOnceDict(PersiDict):
|
|
|
343
336
|
"""
|
|
344
337
|
subdict = self._wrapped_dict.get_subdict(prefix_key)
|
|
345
338
|
result = WriteOnceDict(subdict, self.p_consistency_checks)
|
|
346
|
-
return result
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
# register_parameterizable_class(WriteOnceDict)
|
|
339
|
+
return result
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|