persistent-function-cache 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. persistent_function_cache-0.1.0/LICENSE +21 -0
  2. persistent_function_cache-0.1.0/PKG-INFO +59 -0
  3. persistent_function_cache-0.1.0/README.md +40 -0
  4. persistent_function_cache-0.1.0/pyproject.toml +71 -0
  5. persistent_function_cache-0.1.0/setup.cfg +4 -0
  6. persistent_function_cache-0.1.0/src/persistent_cache/__init__.py +1 -0
  7. persistent_function_cache-0.1.0/src/persistent_cache/caches/__init__.py +0 -0
  8. persistent_function_cache-0.1.0/src/persistent_cache/caches/base.py +4 -0
  9. persistent_function_cache-0.1.0/src/persistent_cache/caches/deep_learning.py +4 -0
  10. persistent_function_cache-0.1.0/src/persistent_cache/caches/speedup_deep_learning.py +4 -0
  11. persistent_function_cache-0.1.0/src/persistent_cache/cli/__init__.py +0 -0
  12. persistent_function_cache-0.1.0/src/persistent_cache/cli/clear_cache.py +53 -0
  13. persistent_function_cache-0.1.0/src/persistent_cache/main/__init__.py +0 -0
  14. persistent_function_cache-0.1.0/src/persistent_cache/main/cacheslot.py +43 -0
  15. persistent_function_cache-0.1.0/src/persistent_cache/main/decorator.py +44 -0
  16. persistent_function_cache-0.1.0/src/persistent_cache/main/hashing.py +68 -0
  17. persistent_function_cache-0.1.0/src/persistent_cache/models/__init__.py +1 -0
  18. persistent_function_cache-0.1.0/src/persistent_cache/models/path.py +19 -0
  19. persistent_function_cache-0.1.0/src/persistent_cache/py.typed +0 -0
  20. persistent_function_cache-0.1.0/src/persistent_cache/reducers/__init__.py +0 -0
  21. persistent_function_cache-0.1.0/src/persistent_cache/reducers/base.py +37 -0
  22. persistent_function_cache-0.1.0/src/persistent_cache/reducers/deep_learning.py +25 -0
  23. persistent_function_cache-0.1.0/src/persistent_cache/reducers/speedup_deep_learning.py +63 -0
  24. persistent_function_cache-0.1.0/src/persistent_function_cache.egg-info/PKG-INFO +59 -0
  25. persistent_function_cache-0.1.0/src/persistent_function_cache.egg-info/SOURCES.txt +29 -0
  26. persistent_function_cache-0.1.0/src/persistent_function_cache.egg-info/dependency_links.txt +1 -0
  27. persistent_function_cache-0.1.0/src/persistent_function_cache.egg-info/entry_points.txt +2 -0
  28. persistent_function_cache-0.1.0/src/persistent_function_cache.egg-info/requires.txt +9 -0
  29. persistent_function_cache-0.1.0/src/persistent_function_cache.egg-info/top_level.txt +1 -0
  30. persistent_function_cache-0.1.0/tests/test_cache.py +26 -0
  31. persistent_function_cache-0.1.0/tests/test_clear_cache.py +8 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2023 Quinten Roets
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,59 @@
1
+ Metadata-Version: 2.1
2
+ Name: persistent-function-cache
3
+ Version: 0.1.0
4
+ Summary: Persistent cache for expensive functions
5
+ Author-email: Quinten Roets <qdr2104@columbia.edu>
6
+ License: MIT
7
+ Project-URL: Source Code, https://github.com/quintenroets/persistent-cache
8
+ Requires-Python: <3.13,>=3.10
9
+ Description-Content-Type: text/markdown
10
+ License-File: LICENSE
11
+ Requires-Dist: package-utils<1,>=0.6.7
12
+ Requires-Dist: powercli<1,>=0.3.2
13
+ Requires-Dist: superpathlib<3,>=2.0.9
14
+ Provides-Extra: dev
15
+ Requires-Dist: package-dev-tools<1,>=0.5.11; extra == "dev"
16
+ Requires-Dist: package-dev-utils<1,>=0.1.6; extra == "dev"
17
+ Requires-Dist: numpy<3,>=1.26.0; extra == "dev"
18
+ Requires-Dist: torch<3,>=1.26.0; extra == "dev"
19
+
20
+ # Cache
21
+ [![PyPI version](https://badge.fury.io/py/persistent-function-cache.svg)](https://badge.fury.io/py/persistent-function-cache)
22
+ ![PyPI downloads](https://img.shields.io/pypi/dm/persistent-function-cache)
23
+ ![Python version](https://img.shields.io/badge/python-3.10--3.12-brightgreen)
24
+ ![Operating system](https://img.shields.io/badge/os-linux%20%7c%20macOS%20%7c%20windows-brightgreen)
25
+ ![Coverage](https://img.shields.io/badge/coverage-88%25-brightgreen)
26
+
27
+ ## Usage
28
+ Use
29
+
30
+ ```shell
31
+ from persistent_cache import cache
32
+
33
+ @cache
34
+ def expensive_function(..):
35
+ ..
36
+
37
+ to cache the result of a function
38
+ ```
39
+
40
+ The cache key for the result is determined by:
41
+ * the function signature
42
+ * the implementation of the function
43
+ * the values of the function arguments
44
+ * custom transformations/reductions can be specified
45
+
46
+ Advantages compared to existing solutions:
47
+ * the cache in invalidated when the behavior of the function changes
48
+ * Each cache value is saved to a separate location. Only values that are effectively needed are loaded.
49
+ * works with function arguments of any complex data type.
50
+ * configurable: custom transformations/reductions can be specified based on the object type.
51
+ * 3 custom transformation groups available out-of-the-box:
52
+ * from persistent_cache import cache
53
+ * from persistent_cache.caches.deep_learning import cache
54
+ * from persistent_cache.caches.speedup_deep_learning import cache`
55
+
56
+ ## Installation
57
+ ```shell
58
+ pip install persistent-function-cache
59
+ ```
@@ -0,0 +1,40 @@
1
+ # Cache
2
+ [![PyPI version](https://badge.fury.io/py/persistent-function-cache.svg)](https://badge.fury.io/py/persistent-function-cache)
3
+ ![PyPI downloads](https://img.shields.io/pypi/dm/persistent-function-cache)
4
+ ![Python version](https://img.shields.io/badge/python-3.10--3.12-brightgreen)
5
+ ![Operating system](https://img.shields.io/badge/os-linux%20%7c%20macOS%20%7c%20windows-brightgreen)
6
+ ![Coverage](https://img.shields.io/badge/coverage-88%25-brightgreen)
7
+
8
+ ## Usage
9
+ Use
10
+
11
+ ```shell
12
+ from persistent_cache import cache
13
+
14
+ @cache
15
+ def expensive_function(..):
16
+ ..
17
+
18
+ to cache the result of a function
19
+ ```
20
+
21
+ The cache key for the result is determined by:
22
+ * the function signature
23
+ * the implementation of the function
24
+ * the values of the function arguments
25
+ * custom transformations/reductions can be specified
26
+
27
+ Advantages compared to existing solutions:
28
+ * the cache in invalidated when the behavior of the function changes
29
+ * Each cache value is saved to a separate location. Only values that are effectively needed are loaded.
30
+ * works with function arguments of any complex data type.
31
+ * configurable: custom transformations/reductions can be specified based on the object type.
32
+ * 3 custom transformation groups available out-of-the-box:
33
+ * from persistent_cache import cache
34
+ * from persistent_cache.caches.deep_learning import cache
35
+ * from persistent_cache.caches.speedup_deep_learning import cache`
36
+
37
+ ## Installation
38
+ ```shell
39
+ pip install persistent-function-cache
40
+ ```
@@ -0,0 +1,71 @@
1
+ [project]
2
+ name = "persistent-function-cache"
3
+ version = "0.1.0"
4
+ description = "Persistent cache for expensive functions"
5
+ authors = [{name = "Quinten Roets", email = "qdr2104@columbia.edu"}]
6
+ license = {text = "MIT"}
7
+ readme = "README.md"
8
+ requires-python = ">=3.10, <3.13"
9
+ dependencies = [
10
+ "package-utils >=0.6.7, <1",
11
+ "powercli >=0.3.2, <1",
12
+ "superpathlib >=2.0.9, <3",
13
+ ]
14
+
15
+ [project.optional-dependencies]
16
+ dev = [
17
+ "package-dev-tools >=0.5.11, <1",
18
+ "package-dev-utils >=0.1.6, <1",
19
+ "numpy >=1.26.0, <3",
20
+ "torch >=1.26.0, <3",
21
+ ]
22
+
23
+ [project.urls]
24
+ "Source Code" = "https://github.com/quintenroets/persistent-cache"
25
+
26
+ [project.scripts]
27
+ clear-persistent-cache = "persistent_cache.cli.clear_cache:entry_point"
28
+
29
+ [build-system]
30
+ requires = ["setuptools"]
31
+ build-backend = "setuptools.build_meta"
32
+
33
+ [tool.coverage.run]
34
+ command_line = "-m pytest tests"
35
+
36
+ [tool.coverage.report]
37
+ precision = 4
38
+ fail_under = 80
39
+
40
+ [tool.mypy]
41
+ strict = true
42
+ no_implicit_reexport = false
43
+
44
+ [tool.pytest.ini_options]
45
+ pythonpath = [
46
+ "src", ".",
47
+ ]
48
+
49
+ [tool.ruff]
50
+ fix = true
51
+
52
+ [tool.ruff.lint]
53
+ select = ["ALL"]
54
+ ignore = [
55
+ "ANN101", # annotate self
56
+ "ANN102", # annotate cls
57
+ "ANN401", # annotated with Any
58
+ "D", # docstrings
59
+ "G004", # logging f-string
60
+ ]
61
+
62
+ [tool.ruff.lint.per-file-ignores]
63
+ "__init__.py" = [
64
+ "F401" # unused import
65
+ ]
66
+ "tests/*" = [
67
+ "S101" # assert used
68
+ ]
69
+
70
+ [tool.setuptools.package-data]
71
+ persistent_cache = ["py.typed"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1 @@
1
+ from .caches.base import cache
@@ -0,0 +1,4 @@
1
+ from persistent_cache.main import decorator
2
+ from persistent_cache.reducers.base import Reducer
3
+
4
+ cache = decorator.cache(Reducer)
@@ -0,0 +1,4 @@
1
+ from persistent_cache.main import decorator
2
+ from persistent_cache.reducers.deep_learning import Reducer
3
+
4
+ cache = decorator.cache(Reducer)
@@ -0,0 +1,4 @@
1
+ from persistent_cache.main import decorator
2
+ from persistent_cache.reducers.speedup_deep_learning import Reducer
3
+
4
+ cache = decorator.cache(Reducer)
@@ -0,0 +1,53 @@
1
+ import time
2
+ from dataclasses import dataclass
3
+ from datetime import datetime, timezone
4
+ from functools import cached_property
5
+ from typing import Annotated
6
+
7
+ import cli
8
+ import typer
9
+ from package_utils.cli import create_entry_point
10
+
11
+ from persistent_cache.models import Path
12
+
13
+
14
+ @dataclass
15
+ class Options:
16
+ max_age: Annotated[
17
+ int | None,
18
+ typer.Option(help="Maximal age of entries to delete"),
19
+ ] = None
20
+ cache_path: Annotated[Path, typer.Option(help="Root of cache directory")] = (
21
+ Path.cache
22
+ )
23
+ verbose: Annotated[bool, typer.Option(help="Show removed entries")] = True
24
+
25
+ @cached_property
26
+ def min_mtime_to_clear(self) -> float | None:
27
+ return time.time() - self.max_age * 60 if self.max_age else None
28
+
29
+
30
+ def main(options: Options) -> None:
31
+ """
32
+ Clear cached values.
33
+ """
34
+
35
+ def should_remove(path_: Path) -> bool:
36
+ return path_.is_file() and (
37
+ options.min_mtime_to_clear is None
38
+ or path_.mtime > options.min_mtime_to_clear
39
+ )
40
+
41
+ with cli.status("Removing.."):
42
+ for path in Options.cache_path.find(should_remove, recurse_on_match=True):
43
+ if options.verbose:
44
+ relative_path = path.relative_to(Options.cache_path)
45
+ timestamp = datetime.fromtimestamp(path.mtime).astimezone(
46
+ tz=timezone.utc,
47
+ )
48
+ message = f"{relative_path} ({timestamp})"
49
+ cli.console.print(message)
50
+ path.unlink()
51
+
52
+
53
+ entry_point = create_entry_point(main)
@@ -0,0 +1,43 @@
1
+ import pickle
2
+ from collections.abc import Callable
3
+ from typing import Any, Generic, TypeVar, cast
4
+
5
+ from persistent_cache.models import Path
6
+ from persistent_cache.reducers.base import Reducer
7
+
8
+ from . import hashing
9
+
10
+ T = TypeVar("T")
11
+
12
+
13
+ class CacheSlot(Generic[T]):
14
+ def __init__(
15
+ self,
16
+ function: Callable[..., T],
17
+ args: tuple[Any, ...],
18
+ kwargs: Any,
19
+ key_reducer: type[Reducer] = Reducer,
20
+ cache_path: Path = Path.cache,
21
+ ) -> None:
22
+ # change cache key when implementation changes
23
+ cache_keys = (function, args, kwargs)
24
+ self.location = (
25
+ cache_path
26
+ / function.__module__.replace(".", "_")
27
+ / function.__name__
28
+ / hashing.compute_hash(key_reducer, cache_keys)
29
+ )
30
+
31
+ @property
32
+ def value(self) -> T:
33
+ try:
34
+ with self.location.open("rb") as fp:
35
+ value = pickle.Unpickler(fp).load() # noqa: S301
36
+ except (pickle.UnpicklingError, EOFError):
37
+ # discard values of corrupted or empty slots
38
+ raise KeyError from None
39
+ return cast(T, value)
40
+
41
+ @value.setter
42
+ def value(self, value: T) -> None:
43
+ self.location.byte_content = pickle.dumps(value)
@@ -0,0 +1,44 @@
1
+ from collections.abc import Callable
2
+ from functools import wraps
3
+ from typing import Any, TypeVar, cast
4
+
5
+ from persistent_cache.models import Path
6
+ from persistent_cache.reducers.base import Reducer
7
+
8
+ from .cacheslot import CacheSlot
9
+
10
+ T = TypeVar("T")
11
+ F = TypeVar("F", bound=Callable[..., Any])
12
+
13
+
14
+ def cache(
15
+ key_reducer: type[Reducer] = Reducer,
16
+ cache_path: Path = Path.cache,
17
+ ) -> Callable[[F], F]:
18
+ """A decorator to cache function results. Decorated functions are only executed if
19
+ result is not present in cache. The arguments of the function can be any nested
20
+ complex object.
21
+
22
+ Use as:
23
+
24
+ from persistent_cache import cache
25
+
26
+ @cache
27
+ def long_function(complex_object):
28
+ ...
29
+ """
30
+
31
+ def cache_decorator(function: F) -> F:
32
+ @wraps(function)
33
+ def wrapped_function(*args: Any, **kwargs: Any) -> Any:
34
+ cache_slot = CacheSlot(function, args, kwargs, key_reducer, cache_path)
35
+ try:
36
+ value = cache_slot.value
37
+ except KeyError:
38
+ value = function(*args, **kwargs)
39
+ cache_slot.value = value
40
+ return value
41
+
42
+ return cast(F, wrapped_function)
43
+
44
+ return cache_decorator
@@ -0,0 +1,68 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ import inspect
5
+ import io
6
+ import pickle
7
+ from typing import TYPE_CHECKING, Any, get_args, get_type_hints
8
+
9
+ from persistent_cache.reducers.base import Reducer
10
+
11
+ if TYPE_CHECKING:
12
+ from collections.abc import Callable, Iterator
13
+ from typing import BinaryIO
14
+
15
+
16
+ class HashPickler(pickle.Pickler):
17
+ def __init__(
18
+ self,
19
+ file_pointer: BinaryIO,
20
+ reducer: type[Reducer] = Reducer,
21
+ ) -> None:
22
+ super().__init__(file_pointer)
23
+ self.reducer = reducer
24
+ self.reducers = {}
25
+ for _, method in inspect.getmembers(reducer, predicate=inspect.ismethod):
26
+ type_hints = get_type_hints(method).values()
27
+ if type_hints:
28
+ argument_type = next(iter(type_hints))
29
+ argument_types = get_args(argument_type) or (argument_type,)
30
+ for argument_type in argument_types:
31
+ self.reducers[argument_type] = method
32
+ self.reducers.pop(Any, None)
33
+
34
+ def reducer_override(self, obj: Any) -> Any:
35
+ """The goal of this pickler is to create hashes of complex objects, not to
36
+ reconstruct complex objects.
37
+
38
+ So mapping does not need to be reversible.
39
+ """
40
+ reducer = next(self.determine_reducer(obj), None)
41
+ reduction: Any
42
+ if reducer is None:
43
+ reduction = NotImplemented
44
+ else:
45
+ mapping = reducer(obj)
46
+ str_mapping = str(object_to_bytes(self.reducer, mapping))
47
+ reduction = str, (str_mapping,)
48
+ return reduction
49
+
50
+ def determine_reducer(self, obj: Any) -> Iterator[Callable[[Any], Any]]:
51
+ if obj is not str:
52
+ for obj_type, reducer in self.reducers.items():
53
+ if isinstance(obj, obj_type):
54
+ yield reducer
55
+
56
+
57
+ def compute_hash(key_reducer: type[Reducer], *args: Any) -> str:
58
+ data = object_to_bytes(key_reducer, args)
59
+ # use fast hash function because it is not used for security
60
+ return hashlib.new("sha1", data=data, usedforsecurity=False).hexdigest()
61
+
62
+
63
+ def object_to_bytes(key_reducer: type[Reducer], args: Any) -> bytes:
64
+ with io.BytesIO() as fp:
65
+ # Use custom pickler to generate bytes from complex structures
66
+ HashPickler(fp, key_reducer).dump(args)
67
+ fp.seek(0)
68
+ return fp.read()
@@ -0,0 +1 @@
1
+ from .path import Path
@@ -0,0 +1,19 @@
1
+ from typing import TypeVar, cast
2
+
3
+ import superpathlib
4
+ from simple_classproperty import classproperty
5
+
6
+ T = TypeVar("T", bound="Path")
7
+
8
+
9
+ class Path(superpathlib.Path):
10
+ @classmethod
11
+ @classproperty
12
+ def source_root(cls: type[T]) -> T:
13
+ return cls(__file__).parent.parent
14
+
15
+ @classmethod
16
+ @classproperty
17
+ def cache(cls: type[T]) -> T:
18
+ path = cls.script_assets / cls.source_root.name
19
+ return cast(T, path)
@@ -0,0 +1,37 @@
1
+ import inspect
2
+ import io
3
+ from types import FunctionType, ModuleType
4
+
5
+
6
+ class Reducer:
7
+ """Inherit from this class to implement own custom pickler.
8
+
9
+ The result of each function are pickled further with their custom
10
+ pickling function, so make sure to reduce each object to a new
11
+ object in each reduction function in order to avoid infinity
12
+ recursive calls.
13
+ """
14
+
15
+ @classmethod
16
+ def reduce_code(cls, code_object: FunctionType | ModuleType | type) -> str:
17
+ """
18
+ custom lambda reduction needed:
19
+ https://www.pythonpool.com/cant-pickle-local-object/
20
+ custom module reduction needed:
21
+ https://stackoverflow.com/questions/2790828/python-cant-pickle-module-objects-error
22
+ name reduction for function/module/class is not enough because we assume
23
+ cache result can change when function/module/class implementation changes
24
+ """
25
+
26
+ try:
27
+ reduction = inspect.getsource(code_object)
28
+ except (TypeError, OSError):
29
+ # cannot access source code of builtins or common libraries
30
+ # but no problem because we assume this code does not change
31
+ reduction = code_object.__name__
32
+ return reduction
33
+
34
+ @classmethod
35
+ def reduce_file_objects(cls, _: io.BytesIO | io.BufferedWriter) -> str:
36
+ # Closed file pointers cannot and should not be pickled for cache functionality
37
+ return ""
@@ -0,0 +1,25 @@
1
+ from typing import Any
2
+
3
+ import torch
4
+ from numpy.typing import NDArray
5
+
6
+ from . import base
7
+
8
+
9
+ class Reducer(base.Reducer):
10
+ @classmethod
11
+ def reduce_model(cls, model: torch.nn.Module) -> tuple[dict[str, Any], Any]:
12
+ """
13
+ Avoid pickling _forward_hooks of model:
14
+ implemented as OrderedDict with nondeterministic keys
15
+ Model outputs determined by:
16
+ - model weights
17
+ - class implementation (forward method)
18
+ """
19
+ return model.state_dict(), model.__class__
20
+
21
+ @classmethod
22
+ def reduce_tensor(cls, tensor: torch.Tensor) -> NDArray[Any]:
23
+ # Pickling a Tensor or a Storage is not deterministic #39382 => convert to numpy
24
+ # https://github.com/pytorch/pytorch/issues/3938
25
+ return tensor.detach().cpu().numpy()
@@ -0,0 +1,63 @@
1
+ import math
2
+ from typing import Any
3
+
4
+ import torch
5
+ from numpy.typing import NDArray
6
+ from torch.utils.data import Dataset
7
+
8
+ from . import deep_learning
9
+
10
+ SEED_VALUE = 493
11
+ LARGE_DIMENSION = 10000
12
+
13
+
14
+ class Reducer(deep_learning.Reducer):
15
+ @classmethod
16
+ def reduce_np_array(cls, array: NDArray[Any]) -> Any:
17
+ shape = array.shape
18
+ reduction: Any
19
+ if shape:
20
+ if math.prod(shape) > LARGE_DIMENSION:
21
+ # only use part of array large for speedup
22
+ length = shape[0] if shape else 0
23
+ data = (array[13**17 % length]) if length > 0 else []
24
+ reduction = shape, data
25
+ else:
26
+ reduction = list(array)
27
+ else:
28
+ reduction = array.item()
29
+ return reduction
30
+
31
+ @classmethod
32
+ def reduce_model(cls, model: torch.nn.Module) -> tuple[Any, Any]:
33
+ weights: Any
34
+ weights, implementation = super().reduce_model(model)
35
+ length = len(weights)
36
+ if length > 0:
37
+ # only use part of weights for speedup
38
+ values = list(weights.values())
39
+ reduction_indices = (0, length // 2, -1)
40
+ weights = tuple(values[i] for i in reduction_indices)
41
+
42
+ return weights, implementation
43
+
44
+ @classmethod
45
+ def reduce_dataset(cls, dataset: Dataset[Any]) -> tuple[int, Any, Any]:
46
+ length = len(dataset) # type: ignore[arg-type]
47
+
48
+ # fix random seed to have deterministic hash
49
+ # for datasets with random augmentation
50
+ torch.random.manual_seed(SEED_VALUE)
51
+
52
+ # only use part of dataset for speedup
53
+ data = dataset[13**17 % length] if length > 0 else []
54
+ if isinstance(data, tuple):
55
+ data, label = data
56
+ else:
57
+ label = None
58
+
59
+ return length, data, label
60
+
61
+ @classmethod
62
+ def reduce_tensor(cls, tensor: torch.Tensor) -> NDArray[Any]:
63
+ return tensor.detach().cpu().numpy()
@@ -0,0 +1,59 @@
1
+ Metadata-Version: 2.1
2
+ Name: persistent-function-cache
3
+ Version: 0.1.0
4
+ Summary: Persistent cache for expensive functions
5
+ Author-email: Quinten Roets <qdr2104@columbia.edu>
6
+ License: MIT
7
+ Project-URL: Source Code, https://github.com/quintenroets/persistent-cache
8
+ Requires-Python: <3.13,>=3.10
9
+ Description-Content-Type: text/markdown
10
+ License-File: LICENSE
11
+ Requires-Dist: package-utils<1,>=0.6.7
12
+ Requires-Dist: powercli<1,>=0.3.2
13
+ Requires-Dist: superpathlib<3,>=2.0.9
14
+ Provides-Extra: dev
15
+ Requires-Dist: package-dev-tools<1,>=0.5.11; extra == "dev"
16
+ Requires-Dist: package-dev-utils<1,>=0.1.6; extra == "dev"
17
+ Requires-Dist: numpy<3,>=1.26.0; extra == "dev"
18
+ Requires-Dist: torch<3,>=1.26.0; extra == "dev"
19
+
20
+ # Cache
21
+ [![PyPI version](https://badge.fury.io/py/persistent-function-cache.svg)](https://badge.fury.io/py/persistent-function-cache)
22
+ ![PyPI downloads](https://img.shields.io/pypi/dm/persistent-function-cache)
23
+ ![Python version](https://img.shields.io/badge/python-3.10--3.12-brightgreen)
24
+ ![Operating system](https://img.shields.io/badge/os-linux%20%7c%20macOS%20%7c%20windows-brightgreen)
25
+ ![Coverage](https://img.shields.io/badge/coverage-88%25-brightgreen)
26
+
27
+ ## Usage
28
+ Use
29
+
30
+ ```shell
31
+ from persistent_cache import cache
32
+
33
+ @cache
34
+ def expensive_function(..):
35
+ ..
36
+
37
+ to cache the result of a function
38
+ ```
39
+
40
+ The cache key for the result is determined by:
41
+ * the function signature
42
+ * the implementation of the function
43
+ * the values of the function arguments
44
+ * custom transformations/reductions can be specified
45
+
46
+ Advantages compared to existing solutions:
47
+ * the cache in invalidated when the behavior of the function changes
48
+ * Each cache value is saved to a separate location. Only values that are effectively needed are loaded.
49
+ * works with function arguments of any complex data type.
50
+ * configurable: custom transformations/reductions can be specified based on the object type.
51
+ * 3 custom transformation groups available out-of-the-box:
52
+ * from persistent_cache import cache
53
+ * from persistent_cache.caches.deep_learning import cache
54
+ * from persistent_cache.caches.speedup_deep_learning import cache`
55
+
56
+ ## Installation
57
+ ```shell
58
+ pip install persistent-function-cache
59
+ ```
@@ -0,0 +1,29 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ src/persistent_cache/__init__.py
5
+ src/persistent_cache/py.typed
6
+ src/persistent_cache/caches/__init__.py
7
+ src/persistent_cache/caches/base.py
8
+ src/persistent_cache/caches/deep_learning.py
9
+ src/persistent_cache/caches/speedup_deep_learning.py
10
+ src/persistent_cache/cli/__init__.py
11
+ src/persistent_cache/cli/clear_cache.py
12
+ src/persistent_cache/main/__init__.py
13
+ src/persistent_cache/main/cacheslot.py
14
+ src/persistent_cache/main/decorator.py
15
+ src/persistent_cache/main/hashing.py
16
+ src/persistent_cache/models/__init__.py
17
+ src/persistent_cache/models/path.py
18
+ src/persistent_cache/reducers/__init__.py
19
+ src/persistent_cache/reducers/base.py
20
+ src/persistent_cache/reducers/deep_learning.py
21
+ src/persistent_cache/reducers/speedup_deep_learning.py
22
+ src/persistent_function_cache.egg-info/PKG-INFO
23
+ src/persistent_function_cache.egg-info/SOURCES.txt
24
+ src/persistent_function_cache.egg-info/dependency_links.txt
25
+ src/persistent_function_cache.egg-info/entry_points.txt
26
+ src/persistent_function_cache.egg-info/requires.txt
27
+ src/persistent_function_cache.egg-info/top_level.txt
28
+ tests/test_cache.py
29
+ tests/test_clear_cache.py
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ clear-persistent-cache = persistent_cache.cli.clear_cache:entry_point
@@ -0,0 +1,9 @@
1
+ package-utils<1,>=0.6.7
2
+ powercli<1,>=0.3.2
3
+ superpathlib<3,>=2.0.9
4
+
5
+ [dev]
6
+ package-dev-tools<1,>=0.5.11
7
+ package-dev-utils<1,>=0.1.6
8
+ numpy<3,>=1.26.0
9
+ torch<3,>=1.26.0
@@ -0,0 +1,26 @@
1
+ import io
2
+ import math
3
+ from collections.abc import Callable
4
+ from typing import Any, TypeVar
5
+
6
+ import cli
7
+ import pytest
8
+
9
+ from persistent_cache.caches import base, deep_learning, speedup_deep_learning
10
+
11
+ F = TypeVar("F", bound=Callable) # type: ignore[type-arg]
12
+
13
+
14
+ def calculate(*args: Any, **kwargs: Any) -> None:
15
+ cli.console.print("calculation started")
16
+ cli.console.print(args, kwargs)
17
+
18
+
19
+ caches = [base.cache, deep_learning.cache, speedup_deep_learning.cache]
20
+
21
+
22
+ @pytest.mark.parametrize("cache", caches)
23
+ def test_cache_with_argument_combination(cache: Callable) -> None: # type: ignore[type-arg]
24
+ cached_function = cache(calculate)
25
+ with io.BytesIO() as fp:
26
+ cached_function(fp, lambda x: x, math, {})
@@ -0,0 +1,8 @@
1
+ from package_dev_utils.tests.args import cli_args
2
+
3
+ from persistent_cache.cli.clear_cache import entry_point
4
+
5
+
6
+ def test_entry_point() -> None:
7
+ with cli_args("--max-age", 0):
8
+ entry_point()