cachier 3.0.1__py3-none-any.whl → 3.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cachier/__init__.py +4 -0
- cachier/config.py +75 -39
- cachier/core.py +19 -19
- cachier/cores/base.py +10 -10
- cachier/cores/memory.py +50 -42
- cachier/cores/mongo.py +25 -28
- cachier/cores/pickle.py +61 -59
- cachier/version.info +1 -1
- {cachier-3.0.1.dist-info → cachier-3.1.1.dist-info}/METADATA +1 -1
- cachier-3.1.1.dist-info/RECORD +19 -0
- {cachier-3.0.1.dist-info → cachier-3.1.1.dist-info}/WHEEL +1 -1
- cachier-3.0.1.dist-info/RECORD +0 -19
- {cachier-3.0.1.dist-info → cachier-3.1.1.dist-info}/LICENSE +0 -0
- {cachier-3.0.1.dist-info → cachier-3.1.1.dist-info}/entry_points.txt +0 -0
- {cachier-3.0.1.dist-info → cachier-3.1.1.dist-info}/top_level.txt +0 -0
cachier/__init__.py
CHANGED
@@ -3,7 +3,9 @@ from .config import (
|
|
3
3
|
disable_caching,
|
4
4
|
enable_caching,
|
5
5
|
get_default_params,
|
6
|
+
get_global_params,
|
6
7
|
set_default_params,
|
8
|
+
set_global_params,
|
7
9
|
)
|
8
10
|
from .core import cachier
|
9
11
|
|
@@ -11,6 +13,8 @@ __all__ = [
|
|
11
13
|
"cachier",
|
12
14
|
"set_default_params",
|
13
15
|
"get_default_params",
|
16
|
+
"set_global_params",
|
17
|
+
"get_global_params",
|
14
18
|
"enable_caching",
|
15
19
|
"disable_caching",
|
16
20
|
]
|
cachier/config.py
CHANGED
@@ -2,7 +2,10 @@ import datetime
|
|
2
2
|
import hashlib
|
3
3
|
import os
|
4
4
|
import pickle
|
5
|
-
|
5
|
+
import threading
|
6
|
+
from collections.abc import Mapping
|
7
|
+
from dataclasses import dataclass, replace
|
8
|
+
from typing import Any, Optional, Union
|
6
9
|
|
7
10
|
from ._types import Backend, HashFunc, Mongetter
|
8
11
|
|
@@ -16,35 +19,36 @@ def _default_hash_func(args, kwds):
|
|
16
19
|
return hashlib.sha256(serialized).hexdigest()
|
17
20
|
|
18
21
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
"
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
22
|
+
@dataclass
|
23
|
+
class Params:
|
24
|
+
"""Default definition for cachier parameters."""
|
25
|
+
|
26
|
+
caching_enabled: bool = True
|
27
|
+
hash_func: HashFunc = _default_hash_func
|
28
|
+
backend: Backend = "pickle"
|
29
|
+
mongetter: Optional[Mongetter] = None
|
30
|
+
stale_after: datetime.timedelta = datetime.timedelta.max
|
31
|
+
next_time: bool = False
|
32
|
+
cache_dir: Union[str, os.PathLike] = "~/.cachier/"
|
33
|
+
pickle_reload: bool = True
|
34
|
+
separate_files: bool = False
|
35
|
+
wait_for_calc_timeout: int = 0
|
36
|
+
allow_none: bool = False
|
37
|
+
|
38
|
+
|
39
|
+
_global_params = Params()
|
40
|
+
|
41
|
+
|
42
|
+
@dataclass
|
43
|
+
class CacheEntry:
|
44
|
+
"""Data class for cache entries."""
|
45
|
+
|
46
|
+
value: Any
|
47
|
+
time: datetime
|
48
|
+
stale: bool
|
49
|
+
_processing: bool
|
50
|
+
_condition: Optional[threading.Condition] = None
|
51
|
+
_completed: bool = False
|
48
52
|
|
49
53
|
|
50
54
|
def _update_with_defaults(
|
@@ -57,11 +61,25 @@ def _update_with_defaults(
|
|
57
61
|
if kw_name in func_kwargs:
|
58
62
|
return func_kwargs.pop(kw_name)
|
59
63
|
if param is None:
|
60
|
-
return cachier.config.
|
64
|
+
return getattr(cachier.config._global_params, name)
|
61
65
|
return param
|
62
66
|
|
63
67
|
|
64
|
-
def set_default_params(**params):
|
68
|
+
def set_default_params(**params: Mapping) -> None:
|
69
|
+
"""Configure default parameters applicable to all memoized functions."""
|
70
|
+
# It is kept for backwards compatibility with desperation warning
|
71
|
+
import warnings
|
72
|
+
|
73
|
+
warnings.warn(
|
74
|
+
"Called `set_default_params` is deprecated and will be removed."
|
75
|
+
" Please use `set_global_params` instead.",
|
76
|
+
DeprecationWarning,
|
77
|
+
stacklevel=2,
|
78
|
+
)
|
79
|
+
set_global_params(**params)
|
80
|
+
|
81
|
+
|
82
|
+
def set_global_params(**params: Mapping) -> None:
|
65
83
|
"""Configure global parameters applicable to all memoized functions.
|
66
84
|
|
67
85
|
This function takes the same keyword parameters as the ones defined in the
|
@@ -76,28 +94,46 @@ def set_default_params(**params):
|
|
76
94
|
"""
|
77
95
|
import cachier
|
78
96
|
|
79
|
-
valid_params =
|
80
|
-
|
97
|
+
valid_params = {
|
98
|
+
k: v
|
99
|
+
for k, v in params.items()
|
100
|
+
if hasattr(cachier.config._global_params, k)
|
101
|
+
}
|
102
|
+
cachier.config._global_params = replace(
|
103
|
+
cachier.config._global_params, **valid_params
|
104
|
+
)
|
105
|
+
|
106
|
+
|
107
|
+
def get_default_params() -> Params:
|
108
|
+
"""Get current set of default parameters."""
|
109
|
+
# It is kept for backwards compatibility with desperation warning
|
110
|
+
import warnings
|
111
|
+
|
112
|
+
warnings.warn(
|
113
|
+
"Called `get_default_params` is deprecated and will be removed."
|
114
|
+
" Please use `get_global_params` instead.",
|
115
|
+
DeprecationWarning,
|
116
|
+
stacklevel=2,
|
81
117
|
)
|
82
|
-
|
118
|
+
return get_global_params()
|
83
119
|
|
84
120
|
|
85
|
-
def
|
121
|
+
def get_global_params() -> Params:
|
86
122
|
"""Get current set of default parameters."""
|
87
123
|
import cachier
|
88
124
|
|
89
|
-
return cachier.config.
|
125
|
+
return cachier.config._global_params
|
90
126
|
|
91
127
|
|
92
128
|
def enable_caching():
|
93
129
|
"""Enable caching globally."""
|
94
130
|
import cachier
|
95
131
|
|
96
|
-
cachier.config.
|
132
|
+
cachier.config._global_params.caching_enabled = True
|
97
133
|
|
98
134
|
|
99
135
|
def disable_caching():
|
100
136
|
"""Disable caching globally."""
|
101
137
|
import cachier
|
102
138
|
|
103
|
-
cachier.config.
|
139
|
+
cachier.config._global_params.caching_enabled = False
|
cachier/core.py
CHANGED
@@ -14,14 +14,13 @@ import warnings
|
|
14
14
|
from collections import OrderedDict
|
15
15
|
from concurrent.futures import ThreadPoolExecutor
|
16
16
|
from functools import wraps
|
17
|
-
from typing import Optional, Union
|
17
|
+
from typing import Any, Optional, Union
|
18
18
|
from warnings import warn
|
19
19
|
|
20
20
|
from .config import (
|
21
21
|
Backend,
|
22
22
|
HashFunc,
|
23
23
|
Mongetter,
|
24
|
-
_default_params,
|
25
24
|
_update_with_defaults,
|
26
25
|
)
|
27
26
|
from .cores.base import RecalculationNeeded, _BaseCore
|
@@ -56,13 +55,11 @@ def _function_thread(core, key, func, args, kwds):
|
|
56
55
|
print(f"Function call failed with the following exception:\n{exc}")
|
57
56
|
|
58
57
|
|
59
|
-
def _calc_entry(core, key, func, args, kwds):
|
58
|
+
def _calc_entry(core, key, func, args, kwds) -> Optional[Any]:
|
59
|
+
core.mark_entry_being_calculated(key)
|
60
60
|
try:
|
61
|
-
core.mark_entry_being_calculated(key)
|
62
|
-
# _get_executor().submit(core.mark_entry_being_calculated, key)
|
63
61
|
func_res = func(*args, **kwds)
|
64
62
|
core.set_entry(key, func_res)
|
65
|
-
# _get_executor().submit(core.set_entry, key, func_res)
|
66
63
|
return func_res
|
67
64
|
finally:
|
68
65
|
core.mark_entry_not_calculated(key)
|
@@ -176,6 +173,8 @@ def cachier(
|
|
176
173
|
None will not be cached and are recalculated every call.
|
177
174
|
|
178
175
|
"""
|
176
|
+
from .config import _global_params
|
177
|
+
|
179
178
|
# Check for deprecated parameters
|
180
179
|
if hash_params is not None:
|
181
180
|
message = (
|
@@ -241,10 +240,9 @@ def cachier(
|
|
241
240
|
func, _is_method=core.func_is_method, args=args, kwds=kwds
|
242
241
|
)
|
243
242
|
|
244
|
-
_print = lambda x: None
|
245
|
-
|
246
|
-
|
247
|
-
if ignore_cache or not _default_params["caching_enabled"]:
|
243
|
+
_print = print if verbose else lambda x: None
|
244
|
+
|
245
|
+
if ignore_cache or not _global_params.caching_enabled:
|
248
246
|
return (
|
249
247
|
func(args[0], **kwargs)
|
250
248
|
if core.func_is_method
|
@@ -253,21 +251,23 @@ def cachier(
|
|
253
251
|
key, entry = core.get_entry((), kwargs)
|
254
252
|
if overwrite_cache:
|
255
253
|
return _calc_entry(core, key, func, args, kwds)
|
256
|
-
if entry is None
|
254
|
+
if entry is None or (
|
255
|
+
not entry._completed and not entry._processing
|
256
|
+
):
|
257
257
|
_print("No entry found. No current calc. Calling like a boss.")
|
258
258
|
return _calc_entry(core, key, func, args, kwds)
|
259
259
|
_print("Entry found.")
|
260
|
-
if _allow_none or entry.
|
260
|
+
if _allow_none or entry.value is not None:
|
261
261
|
_print("Cached result found.")
|
262
262
|
now = datetime.datetime.now()
|
263
|
-
if now - entry
|
263
|
+
if now - entry.time <= _stale_after:
|
264
264
|
_print("And it is fresh!")
|
265
|
-
return entry
|
265
|
+
return entry.value
|
266
266
|
_print("But it is stale... :(")
|
267
|
-
if entry
|
267
|
+
if entry._processing:
|
268
268
|
if _next_time:
|
269
269
|
_print("Returning stale.")
|
270
|
-
return entry
|
270
|
+
return entry.value # return stale val
|
271
271
|
_print("Already calc. Waiting on change.")
|
272
272
|
try:
|
273
273
|
return core.wait_on_entry_calc(key)
|
@@ -275,17 +275,17 @@ def cachier(
|
|
275
275
|
return _calc_entry(core, key, func, args, kwds)
|
276
276
|
if _next_time:
|
277
277
|
_print("Async calc and return stale")
|
278
|
+
core.mark_entry_being_calculated(key)
|
278
279
|
try:
|
279
|
-
core.mark_entry_being_calculated(key)
|
280
280
|
_get_executor().submit(
|
281
281
|
_function_thread, core, key, func, args, kwds
|
282
282
|
)
|
283
283
|
finally:
|
284
284
|
core.mark_entry_not_calculated(key)
|
285
|
-
return entry
|
285
|
+
return entry.value
|
286
286
|
_print("Calling decorated function and waiting")
|
287
287
|
return _calc_entry(core, key, func, args, kwds)
|
288
|
-
if entry
|
288
|
+
if entry._processing:
|
289
289
|
_print("No value but being calculated. Waiting.")
|
290
290
|
try:
|
291
291
|
return core.wait_on_entry_calc(key)
|
cachier/cores/base.py
CHANGED
@@ -9,10 +9,10 @@
|
|
9
9
|
import abc # for the _BaseCore abstract base class
|
10
10
|
import inspect
|
11
11
|
import threading
|
12
|
-
from typing import Callable
|
12
|
+
from typing import Callable, Optional, Tuple
|
13
13
|
|
14
14
|
from .._types import HashFunc
|
15
|
-
from ..config import _update_with_defaults
|
15
|
+
from ..config import CacheEntry, _update_with_defaults
|
16
16
|
|
17
17
|
|
18
18
|
class RecalculationNeeded(Exception):
|
@@ -51,7 +51,7 @@ class _BaseCore:
|
|
51
51
|
"""Return a unique key based on the arguments provided."""
|
52
52
|
return self.hash_func(args, kwds)
|
53
53
|
|
54
|
-
def get_entry(self, args, kwds):
|
54
|
+
def get_entry(self, args, kwds) -> Tuple[str, Optional[CacheEntry]]:
|
55
55
|
"""Get entry based on given arguments.
|
56
56
|
|
57
57
|
Return the result mapped to the given arguments in this core's cache,
|
@@ -76,7 +76,7 @@ class _BaseCore:
|
|
76
76
|
raise RecalculationNeeded()
|
77
77
|
|
78
78
|
@abc.abstractmethod
|
79
|
-
def get_entry_by_key(self, key):
|
79
|
+
def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]:
|
80
80
|
"""Get entry based on given key.
|
81
81
|
|
82
82
|
Return the result mapped to the given key in this core's cache, if such
|
@@ -85,25 +85,25 @@ class _BaseCore:
|
|
85
85
|
"""
|
86
86
|
|
87
87
|
@abc.abstractmethod
|
88
|
-
def set_entry(self, key, func_res):
|
88
|
+
def set_entry(self, key: str, func_res):
|
89
89
|
"""Map the given result to the given key in this core's cache."""
|
90
90
|
|
91
91
|
@abc.abstractmethod
|
92
|
-
def mark_entry_being_calculated(self, key):
|
92
|
+
def mark_entry_being_calculated(self, key: str) -> None:
|
93
93
|
"""Mark the entry mapped by the given key as being calculated."""
|
94
94
|
|
95
95
|
@abc.abstractmethod
|
96
|
-
def mark_entry_not_calculated(self, key):
|
96
|
+
def mark_entry_not_calculated(self, key: str) -> None:
|
97
97
|
"""Mark the entry mapped by the given key as not being calculated."""
|
98
98
|
|
99
99
|
@abc.abstractmethod
|
100
|
-
def wait_on_entry_calc(self, key):
|
100
|
+
def wait_on_entry_calc(self, key: str) -> None:
|
101
101
|
"""Wait on the entry with keys being calculated and returns result."""
|
102
102
|
|
103
103
|
@abc.abstractmethod
|
104
|
-
def clear_cache(self):
|
104
|
+
def clear_cache(self) -> None:
|
105
105
|
"""Clear the cache of this core."""
|
106
106
|
|
107
107
|
@abc.abstractmethod
|
108
|
-
def clear_being_calculated(self):
|
108
|
+
def clear_being_calculated(self) -> None:
|
109
109
|
"""Mark all entries in this cache as not being calculated."""
|
cachier/cores/memory.py
CHANGED
@@ -2,8 +2,10 @@
|
|
2
2
|
|
3
3
|
import threading
|
4
4
|
from datetime import datetime
|
5
|
+
from typing import Any, Optional, Tuple
|
5
6
|
|
6
7
|
from .._types import HashFunc
|
8
|
+
from ..config import CacheEntry
|
7
9
|
from .base import _BaseCore, _get_func_str
|
8
10
|
|
9
11
|
|
@@ -14,76 +16,82 @@ class _MemoryCore(_BaseCore):
|
|
14
16
|
super().__init__(hash_func, wait_for_calc_timeout)
|
15
17
|
self.cache = {}
|
16
18
|
|
17
|
-
def _hash_func_key(self, key):
|
19
|
+
def _hash_func_key(self, key: str) -> str:
|
18
20
|
return f"{_get_func_str(self.func)}:{key}"
|
19
21
|
|
20
|
-
def get_entry_by_key(
|
22
|
+
def get_entry_by_key(
|
23
|
+
self, key: str, reload=False
|
24
|
+
) -> Tuple[str, Optional[CacheEntry]]:
|
21
25
|
with self.lock:
|
22
26
|
return key, self.cache.get(self._hash_func_key(key), None)
|
23
27
|
|
24
|
-
def set_entry(self, key, func_res):
|
28
|
+
def set_entry(self, key: str, func_res: Any) -> None:
|
29
|
+
hash_key = self._hash_func_key(key)
|
25
30
|
with self.lock:
|
26
31
|
try:
|
27
32
|
# we need to retain the existing condition so that
|
28
33
|
# mark_entry_not_calculated can notify all possibly-waiting
|
29
34
|
# threads about it
|
30
|
-
cond = self.cache[
|
35
|
+
cond = self.cache[hash_key]._condition
|
31
36
|
except KeyError: # pragma: no cover
|
32
37
|
cond = None
|
33
|
-
self.cache[
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
38
|
+
self.cache[hash_key] = CacheEntry(
|
39
|
+
value=func_res,
|
40
|
+
time=datetime.now(),
|
41
|
+
stale=False,
|
42
|
+
_processing=False,
|
43
|
+
_condition=cond,
|
44
|
+
_completed=True,
|
45
|
+
)
|
40
46
|
|
41
|
-
def mark_entry_being_calculated(self, key):
|
47
|
+
def mark_entry_being_calculated(self, key: str) -> None:
|
42
48
|
with self.lock:
|
43
49
|
condition = threading.Condition()
|
50
|
+
hash_key = self._hash_func_key(key)
|
51
|
+
if hash_key in self.cache:
|
52
|
+
self.cache[hash_key]._processing = True
|
53
|
+
self.cache[hash_key]._condition = condition
|
44
54
|
# condition.acquire()
|
45
|
-
|
46
|
-
self.cache[
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
"being_calculated": True,
|
54
|
-
"condition": condition,
|
55
|
-
}
|
55
|
+
else:
|
56
|
+
self.cache[hash_key] = CacheEntry(
|
57
|
+
value=None,
|
58
|
+
time=datetime.now(),
|
59
|
+
stale=False,
|
60
|
+
_processing=True,
|
61
|
+
_condition=condition,
|
62
|
+
)
|
56
63
|
|
57
|
-
def mark_entry_not_calculated(self, key):
|
64
|
+
def mark_entry_not_calculated(self, key: str) -> None:
|
65
|
+
hash_key = self._hash_func_key(key)
|
58
66
|
with self.lock:
|
59
|
-
|
60
|
-
entry = self.cache[self._hash_func_key(key)]
|
61
|
-
except KeyError: # pragma: no cover
|
67
|
+
if hash_key not in self.cache:
|
62
68
|
return # that's ok, we don't need an entry in that case
|
63
|
-
entry
|
64
|
-
|
69
|
+
entry = self.cache[hash_key]
|
70
|
+
entry._processing = False
|
71
|
+
cond = entry._condition
|
65
72
|
if cond:
|
66
73
|
cond.acquire()
|
67
74
|
cond.notify_all()
|
68
75
|
cond.release()
|
69
|
-
entry
|
76
|
+
entry._condition = None
|
70
77
|
|
71
|
-
def wait_on_entry_calc(self, key):
|
78
|
+
def wait_on_entry_calc(self, key: str) -> Any:
|
79
|
+
hash_key = self._hash_func_key(key)
|
72
80
|
with self.lock: # pragma: no cover
|
73
|
-
entry = self.cache[
|
74
|
-
if not entry
|
75
|
-
return entry
|
76
|
-
entry
|
77
|
-
entry
|
78
|
-
entry
|
79
|
-
return self.cache[
|
81
|
+
entry = self.cache[hash_key]
|
82
|
+
if not entry._processing:
|
83
|
+
return entry.value
|
84
|
+
entry._condition.acquire()
|
85
|
+
entry._condition.wait()
|
86
|
+
entry._condition.release()
|
87
|
+
return self.cache[hash_key].value
|
80
88
|
|
81
|
-
def clear_cache(self):
|
89
|
+
def clear_cache(self) -> None:
|
82
90
|
with self.lock:
|
83
91
|
self.cache.clear()
|
84
92
|
|
85
|
-
def clear_being_calculated(self):
|
93
|
+
def clear_being_calculated(self) -> None:
|
86
94
|
with self.lock:
|
87
95
|
for entry in self.cache.values():
|
88
|
-
entry
|
89
|
-
entry
|
96
|
+
entry._processing = False
|
97
|
+
entry._condition = None
|
cachier/cores/mongo.py
CHANGED
@@ -13,8 +13,10 @@ import time # to sleep when waiting on Mongo cache\
|
|
13
13
|
import warnings # to warn if pymongo is missing
|
14
14
|
from contextlib import suppress
|
15
15
|
from datetime import datetime
|
16
|
+
from typing import Any, Optional, Tuple
|
16
17
|
|
17
18
|
from .._types import HashFunc, Mongetter
|
19
|
+
from ..config import CacheEntry
|
18
20
|
|
19
21
|
with suppress(ImportError):
|
20
22
|
from bson.binary import Binary # to save binary data to mongodb
|
@@ -65,29 +67,23 @@ class _MongoCore(_BaseCore):
|
|
65
67
|
def _func_str(self) -> str:
|
66
68
|
return _get_func_str(self.func)
|
67
69
|
|
68
|
-
def get_entry_by_key(self, key):
|
70
|
+
def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]:
|
69
71
|
res = self.mongo_collection.find_one(
|
70
72
|
{"func": self._func_str, "key": key}
|
71
73
|
)
|
72
74
|
if not res:
|
73
75
|
return key, None
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
entry = {
|
83
|
-
"value": None,
|
84
|
-
"time": res.get("time", None),
|
85
|
-
"stale": res.get("stale", False),
|
86
|
-
"being_calculated": res.get("being_calculated", False),
|
87
|
-
}
|
76
|
+
val = pickle.loads(res["value"]) if "value" in res else None # noqa: S301
|
77
|
+
entry = CacheEntry(
|
78
|
+
value=val,
|
79
|
+
time=res.get("time", None),
|
80
|
+
stale=res.get("stale", False),
|
81
|
+
_processing=res.get("processing", False),
|
82
|
+
_completed=res.get("completed", False),
|
83
|
+
)
|
88
84
|
return key, entry
|
89
85
|
|
90
|
-
def set_entry(self, key, func_res):
|
86
|
+
def set_entry(self, key: str, func_res: Any) -> None:
|
91
87
|
thebytes = pickle.dumps(func_res)
|
92
88
|
self.mongo_collection.update_one(
|
93
89
|
filter={"func": self._func_str, "key": key},
|
@@ -98,31 +94,32 @@ class _MongoCore(_BaseCore):
|
|
98
94
|
"value": Binary(thebytes),
|
99
95
|
"time": datetime.now(),
|
100
96
|
"stale": False,
|
101
|
-
"
|
97
|
+
"processing": False,
|
98
|
+
"completed": True,
|
102
99
|
}
|
103
100
|
},
|
104
101
|
upsert=True,
|
105
102
|
)
|
106
103
|
|
107
|
-
def mark_entry_being_calculated(self, key):
|
104
|
+
def mark_entry_being_calculated(self, key: str) -> None:
|
108
105
|
self.mongo_collection.update_one(
|
109
106
|
filter={"func": self._func_str, "key": key},
|
110
|
-
update={"$set": {"
|
107
|
+
update={"$set": {"processing": True}},
|
111
108
|
upsert=True,
|
112
109
|
)
|
113
110
|
|
114
|
-
def mark_entry_not_calculated(self, key):
|
111
|
+
def mark_entry_not_calculated(self, key: str) -> None:
|
115
112
|
with suppress(OperationFailure): # don't care in this case
|
116
113
|
self.mongo_collection.update_one(
|
117
114
|
filter={
|
118
115
|
"func": self._func_str,
|
119
116
|
"key": key,
|
120
117
|
},
|
121
|
-
update={"$set": {"
|
118
|
+
update={"$set": {"processing": False}},
|
122
119
|
upsert=False, # should not insert in this case
|
123
120
|
)
|
124
121
|
|
125
|
-
def wait_on_entry_calc(self, key):
|
122
|
+
def wait_on_entry_calc(self, key: str) -> Any:
|
126
123
|
time_spent = 0
|
127
124
|
while True:
|
128
125
|
time.sleep(MONGO_SLEEP_DURATION_IN_SEC)
|
@@ -130,18 +127,18 @@ class _MongoCore(_BaseCore):
|
|
130
127
|
key, entry = self.get_entry_by_key(key)
|
131
128
|
if entry is None:
|
132
129
|
raise RecalculationNeeded()
|
133
|
-
if not entry
|
134
|
-
return entry
|
130
|
+
if not entry._processing:
|
131
|
+
return entry.value
|
135
132
|
self.check_calc_timeout(time_spent)
|
136
133
|
|
137
|
-
def clear_cache(self):
|
134
|
+
def clear_cache(self) -> None:
|
138
135
|
self.mongo_collection.delete_many(filter={"func": self._func_str})
|
139
136
|
|
140
|
-
def clear_being_calculated(self):
|
137
|
+
def clear_being_calculated(self) -> None:
|
141
138
|
self.mongo_collection.update_many(
|
142
139
|
filter={
|
143
140
|
"func": self._func_str,
|
144
|
-
"
|
141
|
+
"processing": True,
|
145
142
|
},
|
146
|
-
update={"$set": {"
|
143
|
+
update={"$set": {"processing": False}},
|
147
144
|
)
|
cachier/cores/pickle.py
CHANGED
@@ -8,15 +8,15 @@
|
|
8
8
|
# Copyright (c) 2016, Shay Palachy <shaypal5@gmail.com>
|
9
9
|
import os
|
10
10
|
import pickle # for local caching
|
11
|
-
from contextlib import suppress
|
12
11
|
from datetime import datetime
|
12
|
+
from typing import Any, Dict, Optional, Tuple
|
13
13
|
|
14
14
|
import portalocker # to lock on pickle cache IO
|
15
15
|
from watchdog.events import PatternMatchingEventHandler
|
16
16
|
from watchdog.observers import Observer
|
17
17
|
|
18
18
|
from .._types import HashFunc
|
19
|
-
from ..config import _update_with_defaults
|
19
|
+
from ..config import CacheEntry, _update_with_defaults
|
20
20
|
|
21
21
|
# Alternative: https://github.com/WoLpH/portalocker
|
22
22
|
from .base import _BaseCore
|
@@ -41,31 +41,28 @@ class _PickleCore(_BaseCore):
|
|
41
41
|
self.observer = None
|
42
42
|
self.value = None
|
43
43
|
|
44
|
-
def inject_observer(self, observer):
|
44
|
+
def inject_observer(self, observer) -> None:
|
45
45
|
"""Inject the observer running this handler."""
|
46
46
|
self.observer = observer
|
47
47
|
|
48
|
-
def _check_calculation(self):
|
49
|
-
# print('checking calc')
|
48
|
+
def _check_calculation(self) -> None:
|
50
49
|
entry = self.core.get_entry_by_key(self.key, True)[1]
|
51
|
-
# print(self.key)
|
52
|
-
# print(entry)
|
53
50
|
try:
|
54
|
-
if not entry
|
51
|
+
if not entry._processing:
|
55
52
|
# print('stopping observer!')
|
56
|
-
self.value = entry
|
53
|
+
self.value = entry.value
|
57
54
|
self.observer.stop()
|
58
55
|
# else:
|
59
|
-
#
|
56
|
+
# print('NOT stopping observer... :(')
|
60
57
|
except TypeError:
|
61
58
|
self.value = None
|
62
59
|
self.observer.stop()
|
63
60
|
|
64
|
-
def on_created(self, event):
|
61
|
+
def on_created(self, event) -> None:
|
65
62
|
"""A Watchdog Event Handler method.""" # noqa: D401
|
66
63
|
self._check_calculation() # pragma: no cover
|
67
64
|
|
68
|
-
def on_modified(self, event):
|
65
|
+
def on_modified(self, event) -> None:
|
69
66
|
"""A Watchdog Event Handler method.""" # noqa: D401
|
70
67
|
self._check_calculation()
|
71
68
|
|
@@ -99,23 +96,23 @@ class _PickleCore(_BaseCore):
|
|
99
96
|
os.path.join(os.path.realpath(self.cache_dir), self.cache_fname)
|
100
97
|
)
|
101
98
|
|
102
|
-
def _reload_cache(self):
|
99
|
+
def _reload_cache(self) -> None:
|
103
100
|
with self.lock:
|
104
101
|
try:
|
105
|
-
with portalocker.Lock(
|
106
|
-
self.
|
107
|
-
) as cache_file:
|
108
|
-
self.cache = pickle.load(cache_file) # noqa: S301
|
102
|
+
with portalocker.Lock(self.cache_fpath, mode="rb") as cf:
|
103
|
+
self.cache = pickle.load(cf) # noqa: S301
|
109
104
|
except (FileNotFoundError, EOFError):
|
110
105
|
self.cache = {}
|
111
106
|
|
112
|
-
def _get_cache(self):
|
107
|
+
def _get_cache(self) -> Dict[str, CacheEntry]:
|
113
108
|
with self.lock:
|
114
109
|
if not self.cache:
|
115
110
|
self._reload_cache()
|
116
111
|
return self.cache
|
117
112
|
|
118
|
-
def _get_cache_by_key(
|
113
|
+
def _get_cache_by_key(
|
114
|
+
self, key=None, hash_str=None
|
115
|
+
) -> Optional[Dict[str, CacheEntry]]:
|
119
116
|
fpath = self.cache_fpath
|
120
117
|
fpath += f"_{hash_str or key}"
|
121
118
|
try:
|
@@ -124,22 +121,24 @@ class _PickleCore(_BaseCore):
|
|
124
121
|
except (FileNotFoundError, EOFError):
|
125
122
|
return None
|
126
123
|
|
127
|
-
def _clear_all_cache_files(self):
|
124
|
+
def _clear_all_cache_files(self) -> None:
|
128
125
|
path, name = os.path.split(self.cache_fpath)
|
129
126
|
for subpath in os.listdir(path):
|
130
127
|
if subpath.startswith(f"{name}_"):
|
131
128
|
os.remove(os.path.join(path, subpath))
|
132
129
|
|
133
|
-
def _clear_being_calculated_all_cache_files(self):
|
130
|
+
def _clear_being_calculated_all_cache_files(self) -> None:
|
134
131
|
path, name = os.path.split(self.cache_fpath)
|
135
132
|
for subpath in os.listdir(path):
|
136
133
|
if subpath.startswith(name):
|
137
134
|
entry = self._get_cache_by_key(hash_str=subpath.split("_")[-1])
|
138
135
|
if entry is not None:
|
139
|
-
entry
|
136
|
+
entry.being_calculated = False
|
140
137
|
self._save_cache(entry, hash_str=subpath.split("_")[-1])
|
141
138
|
|
142
|
-
def _save_cache(
|
139
|
+
def _save_cache(
|
140
|
+
self, cache, key: str = None, hash_str: str = None
|
141
|
+
) -> None:
|
143
142
|
fpath = self.cache_fpath
|
144
143
|
if key is not None:
|
145
144
|
fpath += f"_{key}"
|
@@ -152,7 +151,9 @@ class _PickleCore(_BaseCore):
|
|
152
151
|
if key is None:
|
153
152
|
self._reload_cache()
|
154
153
|
|
155
|
-
def get_entry_by_key(
|
154
|
+
def get_entry_by_key(
|
155
|
+
self, key: str, reload: bool = False
|
156
|
+
) -> Tuple[str, CacheEntry]:
|
156
157
|
with self.lock:
|
157
158
|
if self.separate_files:
|
158
159
|
return key, self._get_cache_by_key(key)
|
@@ -160,13 +161,14 @@ class _PickleCore(_BaseCore):
|
|
160
161
|
self._reload_cache()
|
161
162
|
return key, self._get_cache().get(key, None)
|
162
163
|
|
163
|
-
def set_entry(self, key, func_res):
|
164
|
-
key_data =
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
164
|
+
def set_entry(self, key: str, func_res: Any) -> None:
|
165
|
+
key_data = CacheEntry(
|
166
|
+
value=func_res,
|
167
|
+
time=datetime.now(),
|
168
|
+
stale=False,
|
169
|
+
_processing=False,
|
170
|
+
_completed=True,
|
171
|
+
)
|
170
172
|
if self.separate_files:
|
171
173
|
self._save_cache(key_data, key)
|
172
174
|
return # pragma: no cover
|
@@ -176,51 +178,51 @@ class _PickleCore(_BaseCore):
|
|
176
178
|
cache[key] = key_data
|
177
179
|
self._save_cache(cache)
|
178
180
|
|
179
|
-
def mark_entry_being_calculated_separate_files(self, key):
|
181
|
+
def mark_entry_being_calculated_separate_files(self, key: str) -> None:
|
180
182
|
self._save_cache(
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
183
|
+
CacheEntry(
|
184
|
+
value=None,
|
185
|
+
time=datetime.now(),
|
186
|
+
stale=False,
|
187
|
+
_processing=True,
|
188
|
+
),
|
187
189
|
key=key,
|
188
190
|
)
|
189
191
|
|
190
|
-
def mark_entry_not_calculated_separate_files(self, key):
|
192
|
+
def mark_entry_not_calculated_separate_files(self, key: str) -> None:
|
191
193
|
_, entry = self.get_entry_by_key(key)
|
192
|
-
entry
|
194
|
+
entry._processing = False
|
193
195
|
self._save_cache(entry, key=key)
|
194
196
|
|
195
|
-
def mark_entry_being_calculated(self, key):
|
197
|
+
def mark_entry_being_calculated(self, key: str) -> None:
|
196
198
|
if self.separate_files:
|
197
199
|
self.mark_entry_being_calculated_separate_files(key)
|
198
200
|
return # pragma: no cover
|
199
201
|
|
200
202
|
with self.lock:
|
201
203
|
cache = self._get_cache()
|
202
|
-
|
203
|
-
cache[key]
|
204
|
-
|
205
|
-
cache[key] =
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
204
|
+
if key in cache:
|
205
|
+
cache[key]._processing = True
|
206
|
+
else:
|
207
|
+
cache[key] = CacheEntry(
|
208
|
+
value=None,
|
209
|
+
time=datetime.now(),
|
210
|
+
stale=False,
|
211
|
+
_processing=True,
|
212
|
+
)
|
211
213
|
self._save_cache(cache)
|
212
214
|
|
213
|
-
def mark_entry_not_calculated(self, key):
|
215
|
+
def mark_entry_not_calculated(self, key: str) -> None:
|
214
216
|
if self.separate_files:
|
215
217
|
self.mark_entry_not_calculated_separate_files(key)
|
216
218
|
with self.lock:
|
217
219
|
cache = self._get_cache()
|
218
220
|
# that's ok, we don't need an entry in that case
|
219
|
-
|
220
|
-
cache[key]
|
221
|
+
if isinstance(cache, dict) and key in cache:
|
222
|
+
cache[key]._processing = False
|
221
223
|
self._save_cache(cache)
|
222
224
|
|
223
|
-
def wait_on_entry_calc(self, key):
|
225
|
+
def wait_on_entry_calc(self, key: str) -> Any:
|
224
226
|
if self.separate_files:
|
225
227
|
entry = self._get_cache_by_key(key)
|
226
228
|
filename = f"{self.cache_fname}_{key}"
|
@@ -229,8 +231,8 @@ class _PickleCore(_BaseCore):
|
|
229
231
|
self._reload_cache()
|
230
232
|
entry = self._get_cache()[key]
|
231
233
|
filename = self.cache_fname
|
232
|
-
if not entry
|
233
|
-
return entry
|
234
|
+
if not entry._processing:
|
235
|
+
return entry.value
|
234
236
|
event_handler = _PickleCore.CacheChangeHandler(
|
235
237
|
filename=filename, core=self, key=key
|
236
238
|
)
|
@@ -245,13 +247,13 @@ class _PickleCore(_BaseCore):
|
|
245
247
|
self.check_calc_timeout(time_spent)
|
246
248
|
return event_handler.value
|
247
249
|
|
248
|
-
def clear_cache(self):
|
250
|
+
def clear_cache(self) -> None:
|
249
251
|
if self.separate_files:
|
250
252
|
self._clear_all_cache_files()
|
251
253
|
else:
|
252
254
|
self._save_cache({})
|
253
255
|
|
254
|
-
def clear_being_calculated(self):
|
256
|
+
def clear_being_calculated(self) -> None:
|
255
257
|
if self.separate_files:
|
256
258
|
self._clear_being_calculated_all_cache_files()
|
257
259
|
return # pragma: no cover
|
@@ -259,5 +261,5 @@ class _PickleCore(_BaseCore):
|
|
259
261
|
with self.lock:
|
260
262
|
cache = self._get_cache()
|
261
263
|
for key in cache:
|
262
|
-
cache[key]
|
264
|
+
cache[key]._processing = False
|
263
265
|
self._save_cache(cache)
|
cachier/version.info
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.
|
1
|
+
3.1.1
|
@@ -0,0 +1,19 @@
|
|
1
|
+
cachier/__init__.py,sha256=GZeDebG0EgWIYBmRgPhO19dMiiaam8f9Pu7cWLv3ywY,400
|
2
|
+
cachier/__main__.py,sha256=upg-TlHs1vngKYvkjoPpl3Pvl6xOx4ut-M1mElMiAo0,443
|
3
|
+
cachier/_types.py,sha256=EGJMiw-oCIC_cDLyzw7YC40lfo8jnD3zMmoJpA9Y8Iw,238
|
4
|
+
cachier/_version.py,sha256=yE6UYwvdoIRpw3HmNifiGwV3fqVea5PwZj_EvyosiZ8,1079
|
5
|
+
cachier/config.py,sha256=KOGaXkBRgv66BexENrTMtrC_TYCeV1fA5v8l6Vj2CYI,3840
|
6
|
+
cachier/core.py,sha256=qQa_GT8WQYD-VFcTS8a2v-Hys4_A1no-aM-d3lw1AFY,13149
|
7
|
+
cachier/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
+
cachier/version.info,sha256=bcOy3DE5t9Qhtvwmp_hTOJqbA16k8ukEe5UWBQ3WrHM,6
|
9
|
+
cachier/cores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
10
|
+
cachier/cores/base.py,sha256=jo69c2RMOXbTzDvqRV0UGa5UvyToipv3f62bICIII1k,3631
|
11
|
+
cachier/cores/memory.py,sha256=SSa7qlSU_54YjNYEWrq9rxXozkMYXr5hadAZ3sz62l4,3336
|
12
|
+
cachier/cores/mongo.py,sha256=eRG6XP55G4IcWnoMl5xtDufM1szf8FVbOIBbDH_r-Po,4887
|
13
|
+
cachier/cores/pickle.py,sha256=20c5pg2CS6wAX1PdefCOjl-orec5w7tqEHVqNbZZv0s,9074
|
14
|
+
cachier-3.1.1.dist-info/LICENSE,sha256=-2WrMJkIa0gVP6YQHXXDT7ws-S3M2NEVEF4XF3K8qrY,1069
|
15
|
+
cachier-3.1.1.dist-info/METADATA,sha256=QQxy_bvI6YCcmhvOj0GqWR4Ni31hCEYnDpTJegprWj4,20102
|
16
|
+
cachier-3.1.1.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
|
17
|
+
cachier-3.1.1.dist-info/entry_points.txt,sha256=x4Y7t6Y0Qev_3fgG-Jv7TrsvVdJty3FnGAdkT8-_5mY,49
|
18
|
+
cachier-3.1.1.dist-info/top_level.txt,sha256=_rW_HiJumDCch67YT-WAgzcyvKg5RiYDMZq9d-0ZpaE,8
|
19
|
+
cachier-3.1.1.dist-info/RECORD,,
|
cachier-3.0.1.dist-info/RECORD
DELETED
@@ -1,19 +0,0 @@
|
|
1
|
-
cachier/__init__.py,sha256=6i43oFM_ZTp47b6R8Ws2vvf2JXLvbn-TKzWESwBDb-M,304
|
2
|
-
cachier/__main__.py,sha256=upg-TlHs1vngKYvkjoPpl3Pvl6xOx4ut-M1mElMiAo0,443
|
3
|
-
cachier/_types.py,sha256=EGJMiw-oCIC_cDLyzw7YC40lfo8jnD3zMmoJpA9Y8Iw,238
|
4
|
-
cachier/_version.py,sha256=yE6UYwvdoIRpw3HmNifiGwV3fqVea5PwZj_EvyosiZ8,1079
|
5
|
-
cachier/config.py,sha256=JJutAeOlNjDobNWKteTvcvdj_Pfemk6_dMKIGFtW6UM,2754
|
6
|
-
cachier/core.py,sha256=x0I-Ot2dt7wThu2j8iLXPYvv8exwp4xXMUQIQyImr64,13261
|
7
|
-
cachier/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
-
cachier/version.info,sha256=BuP-SnoYhNINgcxwZcqM_Uw5TsYCxNGA3cObvU99pSE,6
|
9
|
-
cachier/cores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
10
|
-
cachier/cores/base.py,sha256=i6xlR3Em_s6rVER5Uyf1kK86KcaWtb84iKwS9bgDZGI,3465
|
11
|
-
cachier/cores/memory.py,sha256=JFXnXBAeP0nOXOf_vNgpcEuRNjxre8WQjiUBa7yEYmY,3128
|
12
|
-
cachier/cores/mongo.py,sha256=Ez3SNZzAzELO9c_SrGlOX855-5UEPBViabWPjkQnFc0,4917
|
13
|
-
cachier/cores/pickle.py,sha256=fVajLzlhUkQ9ochLWAZnDQQZJPXj0lWl7mgalI-z2x0,8903
|
14
|
-
cachier-3.0.1.dist-info/LICENSE,sha256=-2WrMJkIa0gVP6YQHXXDT7ws-S3M2NEVEF4XF3K8qrY,1069
|
15
|
-
cachier-3.0.1.dist-info/METADATA,sha256=JZCfPStwVwwhYrdvT_Vkmm2FrfqYc2rUGtExGh86DGI,20102
|
16
|
-
cachier-3.0.1.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
|
17
|
-
cachier-3.0.1.dist-info/entry_points.txt,sha256=x4Y7t6Y0Qev_3fgG-Jv7TrsvVdJty3FnGAdkT8-_5mY,49
|
18
|
-
cachier-3.0.1.dist-info/top_level.txt,sha256=_rW_HiJumDCch67YT-WAgzcyvKg5RiYDMZq9d-0ZpaE,8
|
19
|
-
cachier-3.0.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|