persistent-function-cache 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- persistent_function_cache-0.1.0/LICENSE +21 -0
- persistent_function_cache-0.1.0/PKG-INFO +59 -0
- persistent_function_cache-0.1.0/README.md +40 -0
- persistent_function_cache-0.1.0/pyproject.toml +71 -0
- persistent_function_cache-0.1.0/setup.cfg +4 -0
- persistent_function_cache-0.1.0/src/persistent_cache/__init__.py +1 -0
- persistent_function_cache-0.1.0/src/persistent_cache/caches/__init__.py +0 -0
- persistent_function_cache-0.1.0/src/persistent_cache/caches/base.py +4 -0
- persistent_function_cache-0.1.0/src/persistent_cache/caches/deep_learning.py +4 -0
- persistent_function_cache-0.1.0/src/persistent_cache/caches/speedup_deep_learning.py +4 -0
- persistent_function_cache-0.1.0/src/persistent_cache/cli/__init__.py +0 -0
- persistent_function_cache-0.1.0/src/persistent_cache/cli/clear_cache.py +53 -0
- persistent_function_cache-0.1.0/src/persistent_cache/main/__init__.py +0 -0
- persistent_function_cache-0.1.0/src/persistent_cache/main/cacheslot.py +43 -0
- persistent_function_cache-0.1.0/src/persistent_cache/main/decorator.py +44 -0
- persistent_function_cache-0.1.0/src/persistent_cache/main/hashing.py +68 -0
- persistent_function_cache-0.1.0/src/persistent_cache/models/__init__.py +1 -0
- persistent_function_cache-0.1.0/src/persistent_cache/models/path.py +19 -0
- persistent_function_cache-0.1.0/src/persistent_cache/py.typed +0 -0
- persistent_function_cache-0.1.0/src/persistent_cache/reducers/__init__.py +0 -0
- persistent_function_cache-0.1.0/src/persistent_cache/reducers/base.py +37 -0
- persistent_function_cache-0.1.0/src/persistent_cache/reducers/deep_learning.py +25 -0
- persistent_function_cache-0.1.0/src/persistent_cache/reducers/speedup_deep_learning.py +63 -0
- persistent_function_cache-0.1.0/src/persistent_function_cache.egg-info/PKG-INFO +59 -0
- persistent_function_cache-0.1.0/src/persistent_function_cache.egg-info/SOURCES.txt +29 -0
- persistent_function_cache-0.1.0/src/persistent_function_cache.egg-info/dependency_links.txt +1 -0
- persistent_function_cache-0.1.0/src/persistent_function_cache.egg-info/entry_points.txt +2 -0
- persistent_function_cache-0.1.0/src/persistent_function_cache.egg-info/requires.txt +9 -0
- persistent_function_cache-0.1.0/src/persistent_function_cache.egg-info/top_level.txt +1 -0
- persistent_function_cache-0.1.0/tests/test_cache.py +26 -0
- persistent_function_cache-0.1.0/tests/test_clear_cache.py +8 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Quinten Roets
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: persistent-function-cache
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Persistent cache for expensive functions
|
|
5
|
+
Author-email: Quinten Roets <qdr2104@columbia.edu>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Source Code, https://github.com/quintenroets/persistent-cache
|
|
8
|
+
Requires-Python: <3.13,>=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: package-utils<1,>=0.6.7
|
|
12
|
+
Requires-Dist: powercli<1,>=0.3.2
|
|
13
|
+
Requires-Dist: superpathlib<3,>=2.0.9
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: package-dev-tools<1,>=0.5.11; extra == "dev"
|
|
16
|
+
Requires-Dist: package-dev-utils<1,>=0.1.6; extra == "dev"
|
|
17
|
+
Requires-Dist: numpy<3,>=1.26.0; extra == "dev"
|
|
18
|
+
Requires-Dist: torch<3,>=1.26.0; extra == "dev"
|
|
19
|
+
|
|
20
|
+
# Cache
|
|
21
|
+
[](https://badge.fury.io/py/persistent-function-cache)
|
|
22
|
+

|
|
23
|
+

|
|
24
|
+

|
|
25
|
+

|
|
26
|
+
|
|
27
|
+
## Usage
|
|
28
|
+
Use
|
|
29
|
+
|
|
30
|
+
```shell
|
|
31
|
+
from persistent_cache import cache
|
|
32
|
+
|
|
33
|
+
@cache
|
|
34
|
+
def expensive_function(..):
|
|
35
|
+
..
|
|
36
|
+
|
|
37
|
+
to cache the result of a function
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The cache key for the result is determined by:
|
|
41
|
+
* the function signature
|
|
42
|
+
* the implementation of the function
|
|
43
|
+
* the values of the function arguments
|
|
44
|
+
* custom transformations/reductions can be specified
|
|
45
|
+
|
|
46
|
+
Advantages compared to existing solutions:
|
|
47
|
+
* the cache in invalidated when the behavior of the function changes
|
|
48
|
+
* Each cache value is saved to a separate location. Only values that are effectively needed are loaded.
|
|
49
|
+
* works with function arguments of any complex data type.
|
|
50
|
+
* configurable: custom transformations/reductions can be specified based on the object type.
|
|
51
|
+
* 3 custom transformation groups available out-of-the-box:
|
|
52
|
+
* from persistent_cache import cache
|
|
53
|
+
* from persistent_cache.caches.deep_learning import cache
|
|
54
|
+
* from persistent_cache.caches.speedup_deep_learning import cache`
|
|
55
|
+
|
|
56
|
+
## Installation
|
|
57
|
+
```shell
|
|
58
|
+
pip install persistent-function-cache
|
|
59
|
+
```
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Cache
|
|
2
|
+
[](https://badge.fury.io/py/persistent-function-cache)
|
|
3
|
+

|
|
4
|
+

|
|
5
|
+

|
|
6
|
+

|
|
7
|
+
|
|
8
|
+
## Usage
|
|
9
|
+
Use
|
|
10
|
+
|
|
11
|
+
```shell
|
|
12
|
+
from persistent_cache import cache
|
|
13
|
+
|
|
14
|
+
@cache
|
|
15
|
+
def expensive_function(..):
|
|
16
|
+
..
|
|
17
|
+
|
|
18
|
+
to cache the result of a function
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
The cache key for the result is determined by:
|
|
22
|
+
* the function signature
|
|
23
|
+
* the implementation of the function
|
|
24
|
+
* the values of the function arguments
|
|
25
|
+
* custom transformations/reductions can be specified
|
|
26
|
+
|
|
27
|
+
Advantages compared to existing solutions:
|
|
28
|
+
* the cache in invalidated when the behavior of the function changes
|
|
29
|
+
* Each cache value is saved to a separate location. Only values that are effectively needed are loaded.
|
|
30
|
+
* works with function arguments of any complex data type.
|
|
31
|
+
* configurable: custom transformations/reductions can be specified based on the object type.
|
|
32
|
+
* 3 custom transformation groups available out-of-the-box:
|
|
33
|
+
* from persistent_cache import cache
|
|
34
|
+
* from persistent_cache.caches.deep_learning import cache
|
|
35
|
+
* from persistent_cache.caches.speedup_deep_learning import cache`
|
|
36
|
+
|
|
37
|
+
## Installation
|
|
38
|
+
```shell
|
|
39
|
+
pip install persistent-function-cache
|
|
40
|
+
```
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "persistent-function-cache"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Persistent cache for expensive functions"
|
|
5
|
+
authors = [{name = "Quinten Roets", email = "qdr2104@columbia.edu"}]
|
|
6
|
+
license = {text = "MIT"}
|
|
7
|
+
readme = "README.md"
|
|
8
|
+
requires-python = ">=3.10, <3.13"
|
|
9
|
+
dependencies = [
|
|
10
|
+
"package-utils >=0.6.7, <1",
|
|
11
|
+
"powercli >=0.3.2, <1",
|
|
12
|
+
"superpathlib >=2.0.9, <3",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
[project.optional-dependencies]
|
|
16
|
+
dev = [
|
|
17
|
+
"package-dev-tools >=0.5.11, <1",
|
|
18
|
+
"package-dev-utils >=0.1.6, <1",
|
|
19
|
+
"numpy >=1.26.0, <3",
|
|
20
|
+
"torch >=1.26.0, <3",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[project.urls]
|
|
24
|
+
"Source Code" = "https://github.com/quintenroets/persistent-cache"
|
|
25
|
+
|
|
26
|
+
[project.scripts]
|
|
27
|
+
clear-persistent-cache = "persistent_cache.cli.clear_cache:entry_point"
|
|
28
|
+
|
|
29
|
+
[build-system]
|
|
30
|
+
requires = ["setuptools"]
|
|
31
|
+
build-backend = "setuptools.build_meta"
|
|
32
|
+
|
|
33
|
+
[tool.coverage.run]
|
|
34
|
+
command_line = "-m pytest tests"
|
|
35
|
+
|
|
36
|
+
[tool.coverage.report]
|
|
37
|
+
precision = 4
|
|
38
|
+
fail_under = 80
|
|
39
|
+
|
|
40
|
+
[tool.mypy]
|
|
41
|
+
strict = true
|
|
42
|
+
no_implicit_reexport = false
|
|
43
|
+
|
|
44
|
+
[tool.pytest.ini_options]
|
|
45
|
+
pythonpath = [
|
|
46
|
+
"src", ".",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
[tool.ruff]
|
|
50
|
+
fix = true
|
|
51
|
+
|
|
52
|
+
[tool.ruff.lint]
|
|
53
|
+
select = ["ALL"]
|
|
54
|
+
ignore = [
|
|
55
|
+
"ANN101", # annotate self
|
|
56
|
+
"ANN102", # annotate cls
|
|
57
|
+
"ANN401", # annotated with Any
|
|
58
|
+
"D", # docstrings
|
|
59
|
+
"G004", # logging f-string
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
[tool.ruff.lint.per-file-ignores]
|
|
63
|
+
"__init__.py" = [
|
|
64
|
+
"F401" # unused import
|
|
65
|
+
]
|
|
66
|
+
"tests/*" = [
|
|
67
|
+
"S101" # assert used
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
[tool.setuptools.package-data]
|
|
71
|
+
persistent_cache = ["py.typed"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .caches.base import cache
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from datetime import datetime, timezone
|
|
4
|
+
from functools import cached_property
|
|
5
|
+
from typing import Annotated
|
|
6
|
+
|
|
7
|
+
import cli
|
|
8
|
+
import typer
|
|
9
|
+
from package_utils.cli import create_entry_point
|
|
10
|
+
|
|
11
|
+
from persistent_cache.models import Path
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class Options:
|
|
16
|
+
max_age: Annotated[
|
|
17
|
+
int | None,
|
|
18
|
+
typer.Option(help="Maximal age of entries to delete"),
|
|
19
|
+
] = None
|
|
20
|
+
cache_path: Annotated[Path, typer.Option(help="Root of cache directory")] = (
|
|
21
|
+
Path.cache
|
|
22
|
+
)
|
|
23
|
+
verbose: Annotated[bool, typer.Option(help="Show removed entries")] = True
|
|
24
|
+
|
|
25
|
+
@cached_property
|
|
26
|
+
def min_mtime_to_clear(self) -> float | None:
|
|
27
|
+
return time.time() - self.max_age * 60 if self.max_age else None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def main(options: Options) -> None:
|
|
31
|
+
"""
|
|
32
|
+
Clear cached values.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def should_remove(path_: Path) -> bool:
|
|
36
|
+
return path_.is_file() and (
|
|
37
|
+
options.min_mtime_to_clear is None
|
|
38
|
+
or path_.mtime > options.min_mtime_to_clear
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
with cli.status("Removing.."):
|
|
42
|
+
for path in Options.cache_path.find(should_remove, recurse_on_match=True):
|
|
43
|
+
if options.verbose:
|
|
44
|
+
relative_path = path.relative_to(Options.cache_path)
|
|
45
|
+
timestamp = datetime.fromtimestamp(path.mtime).astimezone(
|
|
46
|
+
tz=timezone.utc,
|
|
47
|
+
)
|
|
48
|
+
message = f"{relative_path} ({timestamp})"
|
|
49
|
+
cli.console.print(message)
|
|
50
|
+
path.unlink()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
entry_point = create_entry_point(main)
|
|
File without changes
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import pickle
|
|
2
|
+
from collections.abc import Callable
|
|
3
|
+
from typing import Any, Generic, TypeVar, cast
|
|
4
|
+
|
|
5
|
+
from persistent_cache.models import Path
|
|
6
|
+
from persistent_cache.reducers.base import Reducer
|
|
7
|
+
|
|
8
|
+
from . import hashing
|
|
9
|
+
|
|
10
|
+
T = TypeVar("T")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class CacheSlot(Generic[T]):
|
|
14
|
+
def __init__(
|
|
15
|
+
self,
|
|
16
|
+
function: Callable[..., T],
|
|
17
|
+
args: tuple[Any, ...],
|
|
18
|
+
kwargs: Any,
|
|
19
|
+
key_reducer: type[Reducer] = Reducer,
|
|
20
|
+
cache_path: Path = Path.cache,
|
|
21
|
+
) -> None:
|
|
22
|
+
# change cache key when implementation changes
|
|
23
|
+
cache_keys = (function, args, kwargs)
|
|
24
|
+
self.location = (
|
|
25
|
+
cache_path
|
|
26
|
+
/ function.__module__.replace(".", "_")
|
|
27
|
+
/ function.__name__
|
|
28
|
+
/ hashing.compute_hash(key_reducer, cache_keys)
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def value(self) -> T:
|
|
33
|
+
try:
|
|
34
|
+
with self.location.open("rb") as fp:
|
|
35
|
+
value = pickle.Unpickler(fp).load() # noqa: S301
|
|
36
|
+
except (pickle.UnpicklingError, EOFError):
|
|
37
|
+
# discard values of corrupted or empty slots
|
|
38
|
+
raise KeyError from None
|
|
39
|
+
return cast(T, value)
|
|
40
|
+
|
|
41
|
+
@value.setter
|
|
42
|
+
def value(self, value: T) -> None:
|
|
43
|
+
self.location.byte_content = pickle.dumps(value)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from functools import wraps
|
|
3
|
+
from typing import Any, TypeVar, cast
|
|
4
|
+
|
|
5
|
+
from persistent_cache.models import Path
|
|
6
|
+
from persistent_cache.reducers.base import Reducer
|
|
7
|
+
|
|
8
|
+
from .cacheslot import CacheSlot
|
|
9
|
+
|
|
10
|
+
T = TypeVar("T")
|
|
11
|
+
F = TypeVar("F", bound=Callable[..., Any])
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def cache(
|
|
15
|
+
key_reducer: type[Reducer] = Reducer,
|
|
16
|
+
cache_path: Path = Path.cache,
|
|
17
|
+
) -> Callable[[F], F]:
|
|
18
|
+
"""A decorator to cache function results. Decorated functions are only executed if
|
|
19
|
+
result is not present in cache. The arguments of the function can be any nested
|
|
20
|
+
complex object.
|
|
21
|
+
|
|
22
|
+
Use as:
|
|
23
|
+
|
|
24
|
+
from persistent_cache import cache
|
|
25
|
+
|
|
26
|
+
@cache
|
|
27
|
+
def long_function(complex_object):
|
|
28
|
+
...
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def cache_decorator(function: F) -> F:
|
|
32
|
+
@wraps(function)
|
|
33
|
+
def wrapped_function(*args: Any, **kwargs: Any) -> Any:
|
|
34
|
+
cache_slot = CacheSlot(function, args, kwargs, key_reducer, cache_path)
|
|
35
|
+
try:
|
|
36
|
+
value = cache_slot.value
|
|
37
|
+
except KeyError:
|
|
38
|
+
value = function(*args, **kwargs)
|
|
39
|
+
cache_slot.value = value
|
|
40
|
+
return value
|
|
41
|
+
|
|
42
|
+
return cast(F, wrapped_function)
|
|
43
|
+
|
|
44
|
+
return cache_decorator
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import inspect
|
|
5
|
+
import io
|
|
6
|
+
import pickle
|
|
7
|
+
from typing import TYPE_CHECKING, Any, get_args, get_type_hints
|
|
8
|
+
|
|
9
|
+
from persistent_cache.reducers.base import Reducer
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from collections.abc import Callable, Iterator
|
|
13
|
+
from typing import BinaryIO
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class HashPickler(pickle.Pickler):
|
|
17
|
+
def __init__(
|
|
18
|
+
self,
|
|
19
|
+
file_pointer: BinaryIO,
|
|
20
|
+
reducer: type[Reducer] = Reducer,
|
|
21
|
+
) -> None:
|
|
22
|
+
super().__init__(file_pointer)
|
|
23
|
+
self.reducer = reducer
|
|
24
|
+
self.reducers = {}
|
|
25
|
+
for _, method in inspect.getmembers(reducer, predicate=inspect.ismethod):
|
|
26
|
+
type_hints = get_type_hints(method).values()
|
|
27
|
+
if type_hints:
|
|
28
|
+
argument_type = next(iter(type_hints))
|
|
29
|
+
argument_types = get_args(argument_type) or (argument_type,)
|
|
30
|
+
for argument_type in argument_types:
|
|
31
|
+
self.reducers[argument_type] = method
|
|
32
|
+
self.reducers.pop(Any, None)
|
|
33
|
+
|
|
34
|
+
def reducer_override(self, obj: Any) -> Any:
|
|
35
|
+
"""The goal of this pickler is to create hashes of complex objects, not to
|
|
36
|
+
reconstruct complex objects.
|
|
37
|
+
|
|
38
|
+
So mapping does not need to be reversible.
|
|
39
|
+
"""
|
|
40
|
+
reducer = next(self.determine_reducer(obj), None)
|
|
41
|
+
reduction: Any
|
|
42
|
+
if reducer is None:
|
|
43
|
+
reduction = NotImplemented
|
|
44
|
+
else:
|
|
45
|
+
mapping = reducer(obj)
|
|
46
|
+
str_mapping = str(object_to_bytes(self.reducer, mapping))
|
|
47
|
+
reduction = str, (str_mapping,)
|
|
48
|
+
return reduction
|
|
49
|
+
|
|
50
|
+
def determine_reducer(self, obj: Any) -> Iterator[Callable[[Any], Any]]:
|
|
51
|
+
if obj is not str:
|
|
52
|
+
for obj_type, reducer in self.reducers.items():
|
|
53
|
+
if isinstance(obj, obj_type):
|
|
54
|
+
yield reducer
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def compute_hash(key_reducer: type[Reducer], *args: Any) -> str:
|
|
58
|
+
data = object_to_bytes(key_reducer, args)
|
|
59
|
+
# use fast hash function because it is not used for security
|
|
60
|
+
return hashlib.new("sha1", data=data, usedforsecurity=False).hexdigest()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def object_to_bytes(key_reducer: type[Reducer], args: Any) -> bytes:
|
|
64
|
+
with io.BytesIO() as fp:
|
|
65
|
+
# Use custom pickler to generate bytes from complex structures
|
|
66
|
+
HashPickler(fp, key_reducer).dump(args)
|
|
67
|
+
fp.seek(0)
|
|
68
|
+
return fp.read()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .path import Path
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from typing import TypeVar, cast
|
|
2
|
+
|
|
3
|
+
import superpathlib
|
|
4
|
+
from simple_classproperty import classproperty
|
|
5
|
+
|
|
6
|
+
T = TypeVar("T", bound="Path")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Path(superpathlib.Path):
|
|
10
|
+
@classmethod
|
|
11
|
+
@classproperty
|
|
12
|
+
def source_root(cls: type[T]) -> T:
|
|
13
|
+
return cls(__file__).parent.parent
|
|
14
|
+
|
|
15
|
+
@classmethod
|
|
16
|
+
@classproperty
|
|
17
|
+
def cache(cls: type[T]) -> T:
|
|
18
|
+
path = cls.script_assets / cls.source_root.name
|
|
19
|
+
return cast(T, path)
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
import io
|
|
3
|
+
from types import FunctionType, ModuleType
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Reducer:
|
|
7
|
+
"""Inherit from this class to implement own custom pickler.
|
|
8
|
+
|
|
9
|
+
The result of each function are pickled further with their custom
|
|
10
|
+
pickling function, so make sure to reduce each object to a new
|
|
11
|
+
object in each reduction function in order to avoid infinity
|
|
12
|
+
recursive calls.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
@classmethod
|
|
16
|
+
def reduce_code(cls, code_object: FunctionType | ModuleType | type) -> str:
|
|
17
|
+
"""
|
|
18
|
+
custom lambda reduction needed:
|
|
19
|
+
https://www.pythonpool.com/cant-pickle-local-object/
|
|
20
|
+
custom module reduction needed:
|
|
21
|
+
https://stackoverflow.com/questions/2790828/python-cant-pickle-module-objects-error
|
|
22
|
+
name reduction for function/module/class is not enough because we assume
|
|
23
|
+
cache result can change when function/module/class implementation changes
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
reduction = inspect.getsource(code_object)
|
|
28
|
+
except (TypeError, OSError):
|
|
29
|
+
# cannot access source code of builtins or common libraries
|
|
30
|
+
# but no problem because we assume this code does not change
|
|
31
|
+
reduction = code_object.__name__
|
|
32
|
+
return reduction
|
|
33
|
+
|
|
34
|
+
@classmethod
|
|
35
|
+
def reduce_file_objects(cls, _: io.BytesIO | io.BufferedWriter) -> str:
|
|
36
|
+
# Closed file pointers cannot and should not be pickled for cache functionality
|
|
37
|
+
return ""
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
from numpy.typing import NDArray
|
|
5
|
+
|
|
6
|
+
from . import base
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Reducer(base.Reducer):
|
|
10
|
+
@classmethod
|
|
11
|
+
def reduce_model(cls, model: torch.nn.Module) -> tuple[dict[str, Any], Any]:
|
|
12
|
+
"""
|
|
13
|
+
Avoid pickling _forward_hooks of model:
|
|
14
|
+
implemented as OrderedDict with nondeterministic keys
|
|
15
|
+
Model outputs determined by:
|
|
16
|
+
- model weights
|
|
17
|
+
- class implementation (forward method)
|
|
18
|
+
"""
|
|
19
|
+
return model.state_dict(), model.__class__
|
|
20
|
+
|
|
21
|
+
@classmethod
|
|
22
|
+
def reduce_tensor(cls, tensor: torch.Tensor) -> NDArray[Any]:
|
|
23
|
+
# Pickling a Tensor or a Storage is not deterministic #39382 => convert to numpy
|
|
24
|
+
# https://github.com/pytorch/pytorch/issues/3938
|
|
25
|
+
return tensor.detach().cpu().numpy()
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import math
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
from numpy.typing import NDArray
|
|
6
|
+
from torch.utils.data import Dataset
|
|
7
|
+
|
|
8
|
+
from . import deep_learning
|
|
9
|
+
|
|
10
|
+
SEED_VALUE = 493
|
|
11
|
+
LARGE_DIMENSION = 10000
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Reducer(deep_learning.Reducer):
|
|
15
|
+
@classmethod
|
|
16
|
+
def reduce_np_array(cls, array: NDArray[Any]) -> Any:
|
|
17
|
+
shape = array.shape
|
|
18
|
+
reduction: Any
|
|
19
|
+
if shape:
|
|
20
|
+
if math.prod(shape) > LARGE_DIMENSION:
|
|
21
|
+
# only use part of array large for speedup
|
|
22
|
+
length = shape[0] if shape else 0
|
|
23
|
+
data = (array[13**17 % length]) if length > 0 else []
|
|
24
|
+
reduction = shape, data
|
|
25
|
+
else:
|
|
26
|
+
reduction = list(array)
|
|
27
|
+
else:
|
|
28
|
+
reduction = array.item()
|
|
29
|
+
return reduction
|
|
30
|
+
|
|
31
|
+
@classmethod
|
|
32
|
+
def reduce_model(cls, model: torch.nn.Module) -> tuple[Any, Any]:
|
|
33
|
+
weights: Any
|
|
34
|
+
weights, implementation = super().reduce_model(model)
|
|
35
|
+
length = len(weights)
|
|
36
|
+
if length > 0:
|
|
37
|
+
# only use part of weights for speedup
|
|
38
|
+
values = list(weights.values())
|
|
39
|
+
reduction_indices = (0, length // 2, -1)
|
|
40
|
+
weights = tuple(values[i] for i in reduction_indices)
|
|
41
|
+
|
|
42
|
+
return weights, implementation
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def reduce_dataset(cls, dataset: Dataset[Any]) -> tuple[int, Any, Any]:
|
|
46
|
+
length = len(dataset) # type: ignore[arg-type]
|
|
47
|
+
|
|
48
|
+
# fix random seed to have deterministic hash
|
|
49
|
+
# for datasets with random augmentation
|
|
50
|
+
torch.random.manual_seed(SEED_VALUE)
|
|
51
|
+
|
|
52
|
+
# only use part of dataset for speedup
|
|
53
|
+
data = dataset[13**17 % length] if length > 0 else []
|
|
54
|
+
if isinstance(data, tuple):
|
|
55
|
+
data, label = data
|
|
56
|
+
else:
|
|
57
|
+
label = None
|
|
58
|
+
|
|
59
|
+
return length, data, label
|
|
60
|
+
|
|
61
|
+
@classmethod
|
|
62
|
+
def reduce_tensor(cls, tensor: torch.Tensor) -> NDArray[Any]:
|
|
63
|
+
return tensor.detach().cpu().numpy()
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: persistent-function-cache
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Persistent cache for expensive functions
|
|
5
|
+
Author-email: Quinten Roets <qdr2104@columbia.edu>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Source Code, https://github.com/quintenroets/persistent-cache
|
|
8
|
+
Requires-Python: <3.13,>=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: package-utils<1,>=0.6.7
|
|
12
|
+
Requires-Dist: powercli<1,>=0.3.2
|
|
13
|
+
Requires-Dist: superpathlib<3,>=2.0.9
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: package-dev-tools<1,>=0.5.11; extra == "dev"
|
|
16
|
+
Requires-Dist: package-dev-utils<1,>=0.1.6; extra == "dev"
|
|
17
|
+
Requires-Dist: numpy<3,>=1.26.0; extra == "dev"
|
|
18
|
+
Requires-Dist: torch<3,>=1.26.0; extra == "dev"
|
|
19
|
+
|
|
20
|
+
# Cache
|
|
21
|
+
[](https://badge.fury.io/py/persistent-function-cache)
|
|
22
|
+

|
|
23
|
+

|
|
24
|
+

|
|
25
|
+

|
|
26
|
+
|
|
27
|
+
## Usage
|
|
28
|
+
Use
|
|
29
|
+
|
|
30
|
+
```shell
|
|
31
|
+
from persistent_cache import cache
|
|
32
|
+
|
|
33
|
+
@cache
|
|
34
|
+
def expensive_function(..):
|
|
35
|
+
..
|
|
36
|
+
|
|
37
|
+
to cache the result of a function
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The cache key for the result is determined by:
|
|
41
|
+
* the function signature
|
|
42
|
+
* the implementation of the function
|
|
43
|
+
* the values of the function arguments
|
|
44
|
+
* custom transformations/reductions can be specified
|
|
45
|
+
|
|
46
|
+
Advantages compared to existing solutions:
|
|
47
|
+
* the cache in invalidated when the behavior of the function changes
|
|
48
|
+
* Each cache value is saved to a separate location. Only values that are effectively needed are loaded.
|
|
49
|
+
* works with function arguments of any complex data type.
|
|
50
|
+
* configurable: custom transformations/reductions can be specified based on the object type.
|
|
51
|
+
* 3 custom transformation groups available out-of-the-box:
|
|
52
|
+
* from persistent_cache import cache
|
|
53
|
+
* from persistent_cache.caches.deep_learning import cache
|
|
54
|
+
* from persistent_cache.caches.speedup_deep_learning import cache`
|
|
55
|
+
|
|
56
|
+
## Installation
|
|
57
|
+
```shell
|
|
58
|
+
pip install persistent-function-cache
|
|
59
|
+
```
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/persistent_cache/__init__.py
|
|
5
|
+
src/persistent_cache/py.typed
|
|
6
|
+
src/persistent_cache/caches/__init__.py
|
|
7
|
+
src/persistent_cache/caches/base.py
|
|
8
|
+
src/persistent_cache/caches/deep_learning.py
|
|
9
|
+
src/persistent_cache/caches/speedup_deep_learning.py
|
|
10
|
+
src/persistent_cache/cli/__init__.py
|
|
11
|
+
src/persistent_cache/cli/clear_cache.py
|
|
12
|
+
src/persistent_cache/main/__init__.py
|
|
13
|
+
src/persistent_cache/main/cacheslot.py
|
|
14
|
+
src/persistent_cache/main/decorator.py
|
|
15
|
+
src/persistent_cache/main/hashing.py
|
|
16
|
+
src/persistent_cache/models/__init__.py
|
|
17
|
+
src/persistent_cache/models/path.py
|
|
18
|
+
src/persistent_cache/reducers/__init__.py
|
|
19
|
+
src/persistent_cache/reducers/base.py
|
|
20
|
+
src/persistent_cache/reducers/deep_learning.py
|
|
21
|
+
src/persistent_cache/reducers/speedup_deep_learning.py
|
|
22
|
+
src/persistent_function_cache.egg-info/PKG-INFO
|
|
23
|
+
src/persistent_function_cache.egg-info/SOURCES.txt
|
|
24
|
+
src/persistent_function_cache.egg-info/dependency_links.txt
|
|
25
|
+
src/persistent_function_cache.egg-info/entry_points.txt
|
|
26
|
+
src/persistent_function_cache.egg-info/requires.txt
|
|
27
|
+
src/persistent_function_cache.egg-info/top_level.txt
|
|
28
|
+
tests/test_cache.py
|
|
29
|
+
tests/test_clear_cache.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
persistent_cache
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import io
|
|
2
|
+
import math
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Any, TypeVar
|
|
5
|
+
|
|
6
|
+
import cli
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
from persistent_cache.caches import base, deep_learning, speedup_deep_learning
|
|
10
|
+
|
|
11
|
+
F = TypeVar("F", bound=Callable) # type: ignore[type-arg]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def calculate(*args: Any, **kwargs: Any) -> None:
|
|
15
|
+
cli.console.print("calculation started")
|
|
16
|
+
cli.console.print(args, kwargs)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
caches = [base.cache, deep_learning.cache, speedup_deep_learning.cache]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.mark.parametrize("cache", caches)
|
|
23
|
+
def test_cache_with_argument_combination(cache: Callable) -> None: # type: ignore[type-arg]
|
|
24
|
+
cached_function = cache(calculate)
|
|
25
|
+
with io.BytesIO() as fp:
|
|
26
|
+
cached_function(fp, lambda x: x, math, {})
|