persistent-function-cache 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- persistent_cache/cli/clear_cache.py +1 -3
- persistent_cache/main/cache_slot.py +2 -2
- persistent_cache/main/decorator.py +2 -1
- persistent_cache/main/hashing.py +2 -1
- persistent_cache/models/path.py +1 -3
- persistent_cache/reducers/base.py +8 -10
- persistent_cache/reducers/deep_learning.py +5 -7
- {persistent_function_cache-0.3.0.dist-info → persistent_function_cache-0.3.1.dist-info}/METADATA +4 -4
- {persistent_function_cache-0.3.0.dist-info → persistent_function_cache-0.3.1.dist-info}/RECORD +13 -13
- {persistent_function_cache-0.3.0.dist-info → persistent_function_cache-0.3.1.dist-info}/WHEEL +0 -0
- {persistent_function_cache-0.3.0.dist-info → persistent_function_cache-0.3.1.dist-info}/entry_points.txt +0 -0
- {persistent_function_cache-0.3.0.dist-info → persistent_function_cache-0.3.1.dist-info}/licenses/LICENSE +0 -0
- {persistent_function_cache-0.3.0.dist-info → persistent_function_cache-0.3.1.dist-info}/top_level.txt +0 -0
|
@@ -42,9 +42,7 @@ def main(options: Options) -> None:
|
|
|
42
42
|
for path in Options.cache_path.find(should_remove, recurse_on_match=True):
|
|
43
43
|
if options.verbose:
|
|
44
44
|
relative_path = path.relative_to(Options.cache_path)
|
|
45
|
-
timestamp = datetime.fromtimestamp(path.mtime).astimezone(
|
|
46
|
-
tz=UTC,
|
|
47
|
-
)
|
|
45
|
+
timestamp = datetime.fromtimestamp(path.mtime).astimezone(tz=UTC)
|
|
48
46
|
message = f"{relative_path} ({timestamp})"
|
|
49
47
|
cli.console.print(message)
|
|
50
48
|
path.unlink()
|
|
@@ -30,8 +30,8 @@ class CacheSlot:
|
|
|
30
30
|
try:
|
|
31
31
|
with self.path.open("rb") as fp:
|
|
32
32
|
return pickle.Unpickler(fp).load() # noqa: S301
|
|
33
|
-
except (pickle.UnpicklingError, EOFError):
|
|
34
|
-
# discard values of corrupted or empty slots
|
|
33
|
+
except (FileNotFoundError, pickle.UnpicklingError, EOFError):
|
|
34
|
+
# discard values of missing, corrupted or empty slots
|
|
35
35
|
raise KeyError from None
|
|
36
36
|
|
|
37
37
|
@value.setter
|
|
@@ -49,7 +49,8 @@ def cache( # noqa: PLR0913
|
|
|
49
49
|
deep_learning: bool = False,
|
|
50
50
|
speedup_deep_learning: bool = False,
|
|
51
51
|
) -> F | Callable[[F], F]:
|
|
52
|
-
"""
|
|
52
|
+
"""
|
|
53
|
+
A decorator to cache function results. Decorated functions are only executed if
|
|
53
54
|
result is not present in cache. The arguments of the function can be any nested
|
|
54
55
|
complex object.
|
|
55
56
|
|
persistent_cache/main/hashing.py
CHANGED
|
@@ -30,7 +30,8 @@ class HashPickler(pickle.Pickler):
|
|
|
30
30
|
self.reducers = load_reducers(reducer) # type: ignore[arg-type]
|
|
31
31
|
|
|
32
32
|
def reducer_override(self, obj: Any) -> Any:
|
|
33
|
-
"""
|
|
33
|
+
"""
|
|
34
|
+
The goal of this pickler is to create hashes of complex objects, not to
|
|
34
35
|
reconstruct complex objects.
|
|
35
36
|
|
|
36
37
|
So mapping does not need to be reversible.
|
persistent_cache/models/path.py
CHANGED
|
@@ -4,7 +4,8 @@ from types import FunctionType, ModuleType
|
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class Reducer:
|
|
7
|
-
"""
|
|
7
|
+
"""
|
|
8
|
+
Inherit from this class to implement own custom pickler.
|
|
8
9
|
|
|
9
10
|
The result of each function are pickled further with their custom
|
|
10
11
|
pickling function, so make sure to reduce each object to a new
|
|
@@ -14,15 +15,12 @@ class Reducer:
|
|
|
14
15
|
|
|
15
16
|
@classmethod
|
|
16
17
|
def reduce_code(cls, code_object: FunctionType | ModuleType | type) -> str:
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
cache result can change when function/module/class implementation changes
|
|
24
|
-
"""
|
|
25
|
-
|
|
18
|
+
# name reduction for function/module/class is not enough because we assume
|
|
19
|
+
# cache result can change when function/module/class implementation changes
|
|
20
|
+
# custom lambda reduction needed:
|
|
21
|
+
# https://www.pythonpool.com/cant-pickle-local-object/
|
|
22
|
+
# custom module reduction needed:
|
|
23
|
+
# https://stackoverflow.com/questions/2790828/python-cant-pickle-module-objects-error
|
|
26
24
|
try:
|
|
27
25
|
reduction = inspect.getsource(code_object)
|
|
28
26
|
except (TypeError, OSError):
|
|
@@ -9,13 +9,11 @@ from . import base
|
|
|
9
9
|
class Reducer(base.Reducer):
|
|
10
10
|
@classmethod
|
|
11
11
|
def reduce_model(cls, model: torch.nn.Module) -> tuple[dict[str, Any], Any]:
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
- class implementation (forward method)
|
|
18
|
-
"""
|
|
12
|
+
# avoid pickling _forward_hooks of model:
|
|
13
|
+
# implemented as OrderedDict with nondeterministic keys
|
|
14
|
+
# model outputs determined by:
|
|
15
|
+
# - model weights
|
|
16
|
+
# - class implementation (forward method)
|
|
19
17
|
return model.state_dict(), model.__class__
|
|
20
18
|
|
|
21
19
|
@classmethod
|
{persistent_function_cache-0.3.0.dist-info → persistent_function_cache-0.3.1.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: persistent-function-cache
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Persistent cache for expensive functions
|
|
5
5
|
Author-email: Quinten Roets <qdr2104@columbia.edu>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -8,11 +8,11 @@ Project-URL: Source Code, https://github.com/quintenroets/persistent-cache
|
|
|
8
8
|
Requires-Python: >=3.11
|
|
9
9
|
Description-Content-Type: text/markdown
|
|
10
10
|
License-File: LICENSE
|
|
11
|
-
Requires-Dist: package-utils<1,>=0.
|
|
11
|
+
Requires-Dist: package-utils<1,>=0.9.1
|
|
12
12
|
Requires-Dist: powercli<1,>=0.4.0
|
|
13
|
-
Requires-Dist: superpathlib<3,>=2.0
|
|
13
|
+
Requires-Dist: superpathlib<3,>=2.1.0
|
|
14
14
|
Provides-Extra: dev
|
|
15
|
-
Requires-Dist: package-dev-tools<1,>=0.8.
|
|
15
|
+
Requires-Dist: package-dev-tools<1,>=0.8.7; extra == "dev"
|
|
16
16
|
Requires-Dist: package-dev-utils<1,>=0.1.6; extra == "dev"
|
|
17
17
|
Requires-Dist: numpy<3,>=1.26.0; extra == "dev"
|
|
18
18
|
Requires-Dist: torch<3,>=1.26.0; extra == "dev"
|
{persistent_function_cache-0.3.0.dist-info → persistent_function_cache-0.3.1.dist-info}/RECORD
RENAMED
|
@@ -4,20 +4,20 @@ persistent_cache/caches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG
|
|
|
4
4
|
persistent_cache/caches/deep_learning.py,sha256=tJQrRJK2Sbv7tIGwnrnFAn8nP-Fh7ZgugocKcXeiMnU,126
|
|
5
5
|
persistent_cache/caches/speedup_deep_learning.py,sha256=bywNXeEZsg8Rs2A1_w-ZrJi9Ew2TvNCz59YryGyV0NI,134
|
|
6
6
|
persistent_cache/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
-
persistent_cache/cli/clear_cache.py,sha256=
|
|
7
|
+
persistent_cache/cli/clear_cache.py,sha256=5scYYLGohe_zkS020FoLkwtYuVk5imfxM9dWmjy2V6w,1488
|
|
8
8
|
persistent_cache/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
-
persistent_cache/main/cache_slot.py,sha256=
|
|
10
|
-
persistent_cache/main/decorator.py,sha256=
|
|
11
|
-
persistent_cache/main/hashing.py,sha256=
|
|
9
|
+
persistent_cache/main/cache_slot.py,sha256=dkTIHpHY9OEdPFSJt6PRQSsKJyE3JgrnuyJGNZOnziM,3360
|
|
10
|
+
persistent_cache/main/decorator.py,sha256=pRdVLP4gDPmbmGU4O-fmqDnTNdLyh-6FLG5gkQmOKWw,2691
|
|
11
|
+
persistent_cache/main/hashing.py,sha256=KcOFm5jQN-5gF4s4Y25Eimutvio408ypBSC_sLm3BFA,1900
|
|
12
12
|
persistent_cache/models/__init__.py,sha256=1OeLZ6FZ5gAjlerAjc69Vx0AqWLSx6OSqNCpuzkCZQg,23
|
|
13
|
-
persistent_cache/models/path.py,sha256=
|
|
13
|
+
persistent_cache/models/path.py,sha256=uS0S3DHIYqFm9FVcYnK0f9pwQtrpjrjRC6knZQjcfWU,399
|
|
14
14
|
persistent_cache/reducers/__init__.py,sha256=hCozFuvrAb_gXLcqMMq92v4KdwBbQugwGwCmyLGt5RE,26
|
|
15
|
-
persistent_cache/reducers/base.py,sha256=
|
|
16
|
-
persistent_cache/reducers/deep_learning.py,sha256=
|
|
15
|
+
persistent_cache/reducers/base.py,sha256=yhdpKeoLVBmWPvTrXqcPt7VxqAm8z8l5HuhkXvgarvU,1420
|
|
16
|
+
persistent_cache/reducers/deep_learning.py,sha256=fwJ3Ir4itST2YF246v-0mQ4FklV0Ji9ASWyoxBdjtqo,786
|
|
17
17
|
persistent_cache/reducers/speedup_deep_learning.py,sha256=-kBazhXzPuDpu2CLzw8F2kFyunA1z_H0qU9lW6771y0,1841
|
|
18
|
-
persistent_function_cache-0.3.
|
|
19
|
-
persistent_function_cache-0.3.
|
|
20
|
-
persistent_function_cache-0.3.
|
|
21
|
-
persistent_function_cache-0.3.
|
|
22
|
-
persistent_function_cache-0.3.
|
|
23
|
-
persistent_function_cache-0.3.
|
|
18
|
+
persistent_function_cache-0.3.1.dist-info/licenses/LICENSE,sha256=ENpNaBSvIV7ifD7iNz_-lBhImbiYowqPzBc5V5R8IhM,1070
|
|
19
|
+
persistent_function_cache-0.3.1.dist-info/METADATA,sha256=j96CNcpoNu9_QtmTvZArXHvCaSEnFGd3jnkmFOuwS_I,2185
|
|
20
|
+
persistent_function_cache-0.3.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
21
|
+
persistent_function_cache-0.3.1.dist-info/entry_points.txt,sha256=fSZfuL6Wn42mZldojgGyg7dzSPCRVAuZox3w6dLd2ZI,88
|
|
22
|
+
persistent_function_cache-0.3.1.dist-info/top_level.txt,sha256=Jj-Z9wxHRlRyDUxrfC9GeaV5-8hAf8R4LclQFZesxi8,17
|
|
23
|
+
persistent_function_cache-0.3.1.dist-info/RECORD,,
|
{persistent_function_cache-0.3.0.dist-info → persistent_function_cache-0.3.1.dist-info}/WHEEL
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|