pythonwrench 0.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pythonwrench/__init__.py +490 -0
- pythonwrench/__main__.py +7 -0
- pythonwrench/_core.py +192 -0
- pythonwrench/abc.py +30 -0
- pythonwrench/argparse/__init__.py +81 -0
- pythonwrench/argparse/dataclass_.py +284 -0
- pythonwrench/argparse/parsers.py +619 -0
- pythonwrench/cast.py +247 -0
- pythonwrench/checksum.py +427 -0
- pythonwrench/collections/__init__.py +104 -0
- pythonwrench/collections/collections.py +900 -0
- pythonwrench/collections/prop.py +104 -0
- pythonwrench/collections/reducers.py +330 -0
- pythonwrench/concurrent.py +73 -0
- pythonwrench/csv.py +12 -0
- pythonwrench/dataclasses.py +117 -0
- pythonwrench/datetime.py +17 -0
- pythonwrench/difflib.py +39 -0
- pythonwrench/disk_cache.py +615 -0
- pythonwrench/entrypoints/info.py +44 -0
- pythonwrench/entrypoints/safe_rmdir.py +98 -0
- pythonwrench/entrypoints/tree.py +113 -0
- pythonwrench/enum.py +55 -0
- pythonwrench/functools.py +234 -0
- pythonwrench/hashlib.py +95 -0
- pythonwrench/importlib.py +243 -0
- pythonwrench/inspect.py +69 -0
- pythonwrench/json.py +12 -0
- pythonwrench/jsonl.py +12 -0
- pythonwrench/logging.py +252 -0
- pythonwrench/math.py +107 -0
- pythonwrench/os.py +226 -0
- pythonwrench/pickle.py +12 -0
- pythonwrench/random.py +60 -0
- pythonwrench/re.py +139 -0
- pythonwrench/semver.py +406 -0
- pythonwrench/serialization/__init__.py +70 -0
- pythonwrench/serialization/_core.py +70 -0
- pythonwrench/serialization/csv.py +493 -0
- pythonwrench/serialization/json.py +178 -0
- pythonwrench/serialization/jsonl.py +215 -0
- pythonwrench/serialization/pickle.py +186 -0
- pythonwrench/time.py +34 -0
- pythonwrench/typing/__init__.py +125 -0
- pythonwrench/typing/checks.py +551 -0
- pythonwrench/typing/classes.py +251 -0
- pythonwrench/warnings.py +118 -0
- pythonwrench-0.6.4.dist-info/METADATA +242 -0
- pythonwrench-0.6.4.dist-info/RECORD +52 -0
- pythonwrench-0.6.4.dist-info/WHEEL +4 -0
- pythonwrench-0.6.4.dist-info/entry_points.txt +10 -0
- pythonwrench-0.6.4.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,615 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import time
|
|
8
|
+
import warnings
|
|
9
|
+
from functools import wraps
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import (
|
|
12
|
+
Any,
|
|
13
|
+
Callable,
|
|
14
|
+
Dict,
|
|
15
|
+
Iterable,
|
|
16
|
+
Literal,
|
|
17
|
+
Optional,
|
|
18
|
+
Tuple,
|
|
19
|
+
TypedDict,
|
|
20
|
+
TypeVar,
|
|
21
|
+
Union,
|
|
22
|
+
get_args,
|
|
23
|
+
overload,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
from typing_extensions import ParamSpec
|
|
27
|
+
|
|
28
|
+
from pythonwrench.checksum import checksum_any
|
|
29
|
+
from pythonwrench.datetime import get_now
|
|
30
|
+
from pythonwrench.inspect import get_argnames, get_fullname
|
|
31
|
+
|
|
32
|
+
T = TypeVar("T")
|
|
33
|
+
P = ParamSpec("P")
|
|
34
|
+
U = TypeVar("U")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
ChecksumFn = Callable[[Tuple[Callable[P, T], Tuple, Dict[str, Any]]], int]
|
|
38
|
+
SavingBackend = Literal["csv", "json", "pickle"]
|
|
39
|
+
StoreMode = Literal["outputs_only", "outputs_metadata", "outputs_metadata_inputs"]
|
|
40
|
+
|
|
41
|
+
_DEFAULT_CACHE_STORE_MODE: StoreMode = "outputs_only"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class _CacheMeta(TypedDict):
|
|
45
|
+
datetime: str
|
|
46
|
+
duration: float
|
|
47
|
+
checksum: int
|
|
48
|
+
fn_fullname: str
|
|
49
|
+
output: Any
|
|
50
|
+
input: Optional[Tuple[Any, Any]]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
_DEFAULT_CACHE_DPATH = Path.home().joinpath(".cache", "disk_cache")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
logger = logging.getLogger(__name__)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@overload
|
|
60
|
+
def disk_cache_decorator(
|
|
61
|
+
fn: None = None,
|
|
62
|
+
*,
|
|
63
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
64
|
+
cache_force: bool = False,
|
|
65
|
+
cache_verbose: int = 0,
|
|
66
|
+
cache_checksum_fn: ChecksumFn = checksum_any,
|
|
67
|
+
cache_saving_backend: Literal["custom"],
|
|
68
|
+
cache_fname_fmt: Union[
|
|
69
|
+
str, Callable[..., str]
|
|
70
|
+
] = "{fn_name}_{checksum_hex}{suffix}",
|
|
71
|
+
cache_fname_fmt_args: Optional[Iterable[str]] = None,
|
|
72
|
+
cache_dump_fn: Callable[[Any, Path], Any],
|
|
73
|
+
cache_load_fn: Callable[[Path], Any],
|
|
74
|
+
cache_enable: bool = True,
|
|
75
|
+
cache_store_mode: StoreMode,
|
|
76
|
+
) -> Callable[[Callable[P, T]], Callable[P, T]]:
|
|
77
|
+
"""Perform the disk cache decorator operation."""
|
|
78
|
+
...
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@overload
|
|
82
|
+
def disk_cache_decorator(
|
|
83
|
+
fn: None = None,
|
|
84
|
+
*,
|
|
85
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
86
|
+
cache_force: bool = False,
|
|
87
|
+
cache_verbose: int = 0,
|
|
88
|
+
cache_checksum_fn: ChecksumFn = checksum_any,
|
|
89
|
+
cache_saving_backend: SavingBackend,
|
|
90
|
+
cache_fname_fmt: Union[
|
|
91
|
+
str, Callable[..., str]
|
|
92
|
+
] = "{fn_name}_{checksum_hex}{suffix}",
|
|
93
|
+
cache_fname_fmt_args: Optional[Iterable[str]] = None,
|
|
94
|
+
cache_dump_fn: None = None,
|
|
95
|
+
cache_load_fn: None = None,
|
|
96
|
+
cache_enable: bool = True,
|
|
97
|
+
cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
|
|
98
|
+
) -> Callable[[Callable[P, T]], Callable[P, T]]:
|
|
99
|
+
"""Perform the disk cache decorator operation."""
|
|
100
|
+
...
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@overload
|
|
104
|
+
def disk_cache_decorator(
|
|
105
|
+
fn: None = None,
|
|
106
|
+
*,
|
|
107
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
108
|
+
cache_force: bool = False,
|
|
109
|
+
cache_verbose: int = 0,
|
|
110
|
+
cache_checksum_fn: ChecksumFn = checksum_any,
|
|
111
|
+
cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto",
|
|
112
|
+
cache_fname_fmt: Union[
|
|
113
|
+
str, Callable[..., str]
|
|
114
|
+
] = "{fn_name}_{checksum_hex}{suffix}",
|
|
115
|
+
cache_fname_fmt_args: Optional[Iterable[str]] = None,
|
|
116
|
+
cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None,
|
|
117
|
+
cache_load_fn: Optional[Callable[[Path], Any]] = None,
|
|
118
|
+
cache_enable: bool = True,
|
|
119
|
+
cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
|
|
120
|
+
) -> Callable[[Callable[P, T]], Callable[P, T]]:
|
|
121
|
+
"""Perform the disk cache decorator operation."""
|
|
122
|
+
...
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@overload
|
|
126
|
+
def disk_cache_decorator(
|
|
127
|
+
fn: Callable[P, T],
|
|
128
|
+
*,
|
|
129
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
130
|
+
cache_force: bool = False,
|
|
131
|
+
cache_verbose: int = 0,
|
|
132
|
+
cache_checksum_fn: ChecksumFn = checksum_any,
|
|
133
|
+
cache_saving_backend: Literal["custom"],
|
|
134
|
+
cache_fname_fmt: Union[
|
|
135
|
+
str, Callable[..., str]
|
|
136
|
+
] = "{fn_name}_{checksum_hex}{suffix}",
|
|
137
|
+
cache_fname_fmt_args: Optional[Iterable[str]] = None,
|
|
138
|
+
cache_dump_fn: Callable[[Any, Path], Any],
|
|
139
|
+
cache_load_fn: Callable[[Path], Any],
|
|
140
|
+
cache_enable: bool = True,
|
|
141
|
+
cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
|
|
142
|
+
) -> Callable[P, T]:
|
|
143
|
+
"""Perform the disk cache decorator operation."""
|
|
144
|
+
...
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@overload
|
|
148
|
+
def disk_cache_decorator(
|
|
149
|
+
fn: Callable[P, T],
|
|
150
|
+
*,
|
|
151
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
152
|
+
cache_force: bool = False,
|
|
153
|
+
cache_verbose: int = 0,
|
|
154
|
+
cache_checksum_fn: ChecksumFn = checksum_any,
|
|
155
|
+
cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto",
|
|
156
|
+
cache_fname_fmt: Union[
|
|
157
|
+
str, Callable[..., str]
|
|
158
|
+
] = "{fn_name}_{checksum_hex}{suffix}",
|
|
159
|
+
cache_fname_fmt_args: Optional[Iterable[str]] = None,
|
|
160
|
+
cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None,
|
|
161
|
+
cache_load_fn: Optional[Callable[[Path], Any]] = None,
|
|
162
|
+
cache_enable: bool = True,
|
|
163
|
+
cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
|
|
164
|
+
) -> Callable[P, T]:
|
|
165
|
+
"""Perform the disk cache decorator operation."""
|
|
166
|
+
...
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def disk_cache_decorator(
|
|
170
|
+
fn: Optional[Callable[P, T]] = None,
|
|
171
|
+
*,
|
|
172
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
173
|
+
cache_force: bool = False,
|
|
174
|
+
cache_verbose: int = 0,
|
|
175
|
+
cache_checksum_fn: ChecksumFn = checksum_any,
|
|
176
|
+
cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto",
|
|
177
|
+
cache_fname_fmt: Union[
|
|
178
|
+
str, Callable[..., str]
|
|
179
|
+
] = "{fn_name}_{checksum_hex}{suffix}",
|
|
180
|
+
cache_fname_fmt_args: Optional[Iterable[str]] = None,
|
|
181
|
+
cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None,
|
|
182
|
+
cache_load_fn: Optional[Callable[[Path], Any]] = None,
|
|
183
|
+
cache_enable: bool = True,
|
|
184
|
+
cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
|
|
185
|
+
) -> Callable:
|
|
186
|
+
"""Decorator to store function output in a cache file.
|
|
187
|
+
|
|
188
|
+
Cache file is identified by the checksum of the function arguments, and stored by default in `"~/.cache/disk_cache/<Function_name>/"` directory.
|
|
189
|
+
|
|
190
|
+
Example
|
|
191
|
+
-------
|
|
192
|
+
>>> import pythonwrench as pw
|
|
193
|
+
>>> @pw.disk_cache_decorator
|
|
194
|
+
>>> def heavy_processing():
|
|
195
|
+
>>> # Lot of stuff here
|
|
196
|
+
>>> ...
|
|
197
|
+
>>> outputs = heavy_processing() # first time function is called
|
|
198
|
+
>>> outputs = heavy_processing() # second time outputs is loaded from disk
|
|
199
|
+
|
|
200
|
+
Args:
|
|
201
|
+
fn: Function to store its output. By default, it must be a callable that returns a pickable object.
|
|
202
|
+
cache_dpath: Cache directory path. defaults to `"~/.cache/disk_cache"`.
|
|
203
|
+
cache_force: Force function call and overwrite cache. defaults to False.
|
|
204
|
+
cache_verbose: Set verbose logging level. Higher means more verbose. defaults to 0.
|
|
205
|
+
cache_checksum_fn: Checksum function to identify input arguments. defaults to ``pythonwrench.checksum_any``.
|
|
206
|
+
cache_saving_backend: Optional saving backend. Can be one of ('csv', 'json', 'pickle', 'custom', 'auto'). defaults to 'auto'.
|
|
207
|
+
cache_fname_fmt: Cache filename format. defaults to "{fn_name}_{checksum_hex}{suffix}".
|
|
208
|
+
cache_dump_fn: Dump/save function to store outputs and overwrite saving backend. defaults to None.
|
|
209
|
+
cache_load_fn: Load function to store outputs and overwrite saving backend. defaults to None.
|
|
210
|
+
cache_enable: Enable disk cache. If False, the function has no effect. defaults to True.
|
|
211
|
+
cache_store_mode: Disk cache storage mode. By default, it store function output and saved date into the cache file. defaults to 'outputs_metadata'.
|
|
212
|
+
"""
|
|
213
|
+
impl_fn = _disk_cache_impl(
|
|
214
|
+
cache_dpath=cache_dpath,
|
|
215
|
+
cache_force=cache_force,
|
|
216
|
+
cache_verbose=cache_verbose,
|
|
217
|
+
cache_checksum_fn=cache_checksum_fn,
|
|
218
|
+
cache_saving_backend=cache_saving_backend,
|
|
219
|
+
cache_fname_fmt=cache_fname_fmt,
|
|
220
|
+
cache_fname_fmt_args=cache_fname_fmt_args,
|
|
221
|
+
cache_dump_fn=cache_dump_fn,
|
|
222
|
+
cache_load_fn=cache_load_fn,
|
|
223
|
+
cache_enable=cache_enable,
|
|
224
|
+
cache_store_mode=cache_store_mode,
|
|
225
|
+
)
|
|
226
|
+
if fn is not None:
|
|
227
|
+
return impl_fn(fn)
|
|
228
|
+
else:
|
|
229
|
+
return impl_fn
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
@overload
|
|
233
|
+
def disk_cache_call(
|
|
234
|
+
fn: Callable[..., T],
|
|
235
|
+
*args,
|
|
236
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
237
|
+
cache_force: bool = False,
|
|
238
|
+
cache_verbose: int = 0,
|
|
239
|
+
cache_checksum_fn: ChecksumFn = checksum_any,
|
|
240
|
+
cache_saving_backend: Literal["custom"],
|
|
241
|
+
cache_fname_fmt: Union[
|
|
242
|
+
str, Callable[..., str]
|
|
243
|
+
] = "{fn_name}_{checksum_hex}{suffix}",
|
|
244
|
+
cache_fname_fmt_args: Optional[Iterable[str]] = None,
|
|
245
|
+
cache_dump_fn: Callable[[Any, Path], Any],
|
|
246
|
+
cache_load_fn: Callable[[Path], Any],
|
|
247
|
+
cache_enable: bool = True,
|
|
248
|
+
cache_store_mode: StoreMode,
|
|
249
|
+
**kwargs,
|
|
250
|
+
) -> T:
|
|
251
|
+
"""Perform the disk cache call operation."""
|
|
252
|
+
...
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
@overload
|
|
256
|
+
def disk_cache_call(
|
|
257
|
+
fn: Callable[..., T],
|
|
258
|
+
*args,
|
|
259
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
260
|
+
cache_force: bool = False,
|
|
261
|
+
cache_verbose: int = 0,
|
|
262
|
+
cache_checksum_fn: ChecksumFn = checksum_any,
|
|
263
|
+
cache_saving_backend: SavingBackend,
|
|
264
|
+
cache_fname_fmt: Union[
|
|
265
|
+
str, Callable[..., str]
|
|
266
|
+
] = "{fn_name}_{checksum_hex}{suffix}",
|
|
267
|
+
cache_fname_fmt_args: Optional[Iterable[str]] = None,
|
|
268
|
+
cache_dump_fn: None = None,
|
|
269
|
+
cache_load_fn: None = None,
|
|
270
|
+
cache_enable: bool = True,
|
|
271
|
+
cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
|
|
272
|
+
**kwargs,
|
|
273
|
+
) -> T:
|
|
274
|
+
"""Perform the disk cache call operation."""
|
|
275
|
+
...
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
@overload
|
|
279
|
+
def disk_cache_call(
|
|
280
|
+
fn: Callable[..., T],
|
|
281
|
+
*args,
|
|
282
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
283
|
+
cache_force: bool = False,
|
|
284
|
+
cache_verbose: int = 0,
|
|
285
|
+
cache_checksum_fn: ChecksumFn = checksum_any,
|
|
286
|
+
cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto",
|
|
287
|
+
cache_fname_fmt: Union[
|
|
288
|
+
str, Callable[..., str]
|
|
289
|
+
] = "{fn_name}_{checksum_hex}{suffix}",
|
|
290
|
+
cache_fname_fmt_args: Optional[Iterable[str]] = None,
|
|
291
|
+
cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None,
|
|
292
|
+
cache_load_fn: Optional[Callable[[Path], Any]] = None,
|
|
293
|
+
cache_enable: bool = True,
|
|
294
|
+
cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
|
|
295
|
+
**kwargs,
|
|
296
|
+
) -> T:
|
|
297
|
+
"""Perform the disk cache call operation."""
|
|
298
|
+
...
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def disk_cache_call(
|
|
302
|
+
fn: Callable[..., T],
|
|
303
|
+
*args,
|
|
304
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
305
|
+
cache_force: bool = False,
|
|
306
|
+
cache_verbose: int = 0,
|
|
307
|
+
cache_checksum_fn: ChecksumFn = checksum_any,
|
|
308
|
+
cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto",
|
|
309
|
+
cache_fname_fmt: Union[
|
|
310
|
+
str, Callable[..., str]
|
|
311
|
+
] = "{fn_name}_{checksum_hex}{suffix}",
|
|
312
|
+
cache_fname_fmt_args: Optional[Iterable[str]] = None,
|
|
313
|
+
cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None,
|
|
314
|
+
cache_load_fn: Optional[Callable[[Path], Any]] = None,
|
|
315
|
+
cache_enable: bool = True,
|
|
316
|
+
cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
|
|
317
|
+
**kwargs,
|
|
318
|
+
) -> T:
|
|
319
|
+
r"""Call function and store output in a cache file.
|
|
320
|
+
|
|
321
|
+
Cache file is identified by the checksum of the function arguments, and stored by default in '~/.cache/disk_cache/<Function_name>/' directory.
|
|
322
|
+
|
|
323
|
+
Example
|
|
324
|
+
-------
|
|
325
|
+
>>> import pythonwrench as pw
|
|
326
|
+
>>> def heavy_processing():
|
|
327
|
+
>>> # Lot of stuff here
|
|
328
|
+
>>> ...
|
|
329
|
+
>>> outputs = pw.disk_cache_call(heavy_processing) # first time function is called
|
|
330
|
+
>>> outputs = pw.disk_cache_call(heavy_processing) # second time outputs is loaded from disk
|
|
331
|
+
|
|
332
|
+
Args:
|
|
333
|
+
fn: Function to store its output. By default, it must be a callable that returns a pickable object.
|
|
334
|
+
cache_dpath: Cache directory path. defaults to '~/.cache/disk_cache'.
|
|
335
|
+
cache_force: Force function call and overwrite cache. defaults to False.
|
|
336
|
+
cache_verbose: Set verbose logging level. Higher means more verbose. defaults to 0.
|
|
337
|
+
cache_checksum_fn: Checksum function to identify input arguments. defaults to ``pythonwrench.checksum_any``.
|
|
338
|
+
cache_saving_backend: Optional saving backend. Can be one of ('csv', 'json', 'pickle', 'custom', 'auto'). defaults to 'auto'.
|
|
339
|
+
cache_fname_fmt: Cache filename format. defaults to '{fn_name}_{checksum_hex}{suffix}'.
|
|
340
|
+
cache_dump_fn: Dump/save function to store outputs and overwrite saving backend. defaults to None.
|
|
341
|
+
cache_load_fn: Load function to store outputs and overwrite saving backend. defaults to None.
|
|
342
|
+
cache_enable: Enable disk cache. If False, the function has no effect. defaults to True.
|
|
343
|
+
cache_store_mode: Disk cache storage mode. By default, it store function output and saved date into the cache file. defaults to 'outputs_metadata'.
|
|
344
|
+
\*args: Positional arguments passed to the function.
|
|
345
|
+
\*\*kwargs: Keywords arguments passed to the function.
|
|
346
|
+
"""
|
|
347
|
+
wrapped_fn = _disk_cache_impl(
|
|
348
|
+
cache_dpath=cache_dpath,
|
|
349
|
+
cache_force=cache_force,
|
|
350
|
+
cache_verbose=cache_verbose,
|
|
351
|
+
cache_checksum_fn=cache_checksum_fn,
|
|
352
|
+
cache_saving_backend=cache_saving_backend,
|
|
353
|
+
cache_fname_fmt=cache_fname_fmt,
|
|
354
|
+
cache_fname_fmt_args=cache_fname_fmt_args,
|
|
355
|
+
cache_dump_fn=cache_dump_fn,
|
|
356
|
+
cache_load_fn=cache_load_fn,
|
|
357
|
+
cache_enable=cache_enable,
|
|
358
|
+
cache_store_mode=cache_store_mode,
|
|
359
|
+
)
|
|
360
|
+
return wrapped_fn(fn)(*args, **kwargs)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _disk_cache_impl(
|
|
364
|
+
*,
|
|
365
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
366
|
+
cache_force: bool = False,
|
|
367
|
+
cache_verbose: int = 0,
|
|
368
|
+
cache_checksum_fn: ChecksumFn = checksum_any,
|
|
369
|
+
cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto",
|
|
370
|
+
cache_fname_fmt: Union[
|
|
371
|
+
str, Callable[..., str]
|
|
372
|
+
] = "{fn_name}_{checksum_hex}{suffix}",
|
|
373
|
+
cache_fname_fmt_args: Optional[Iterable[str]] = None,
|
|
374
|
+
cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None,
|
|
375
|
+
cache_load_fn: Optional[Callable[[Path], Any]] = None,
|
|
376
|
+
cache_enable: bool = True,
|
|
377
|
+
cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
|
|
378
|
+
) -> Callable[[Callable[P, T]], Callable[P, T]]:
|
|
379
|
+
# for backward compatibility
|
|
380
|
+
"""Perform the disk cache impl operation."""
|
|
381
|
+
if cache_fname_fmt is None:
|
|
382
|
+
expected = "{fn_name}_{csum}{suffix}"
|
|
383
|
+
msg = f"Deprecated argument value {cache_fname_fmt=}. (use {expected} instead)"
|
|
384
|
+
warnings.warn(msg, DeprecationWarning)
|
|
385
|
+
cache_fname_fmt = expected
|
|
386
|
+
|
|
387
|
+
if cache_saving_backend is None:
|
|
388
|
+
expected = "auto"
|
|
389
|
+
msg = f"Deprecated argument value {cache_saving_backend=}. (use {expected} instead)"
|
|
390
|
+
warnings.warn(msg, DeprecationWarning)
|
|
391
|
+
cache_saving_backend = expected
|
|
392
|
+
|
|
393
|
+
if cache_saving_backend == "auto":
|
|
394
|
+
if cache_dump_fn is not None and cache_load_fn is not None:
|
|
395
|
+
cache_saving_backend = "custom"
|
|
396
|
+
else:
|
|
397
|
+
cache_saving_backend = "pickle"
|
|
398
|
+
|
|
399
|
+
if cache_saving_backend == "pickle":
|
|
400
|
+
from pythonwrench.serialization.pickle import dump_pickle, load_pickle
|
|
401
|
+
|
|
402
|
+
suffix = ".pickle"
|
|
403
|
+
cache_dump_fn = dump_pickle
|
|
404
|
+
cache_load_fn = load_pickle
|
|
405
|
+
|
|
406
|
+
elif cache_saving_backend == "json":
|
|
407
|
+
from pythonwrench.serialization.json import dump_json, load_json
|
|
408
|
+
|
|
409
|
+
suffix = ".json"
|
|
410
|
+
cache_dump_fn = dump_json
|
|
411
|
+
cache_load_fn = load_json
|
|
412
|
+
|
|
413
|
+
elif cache_saving_backend == "csv":
|
|
414
|
+
from pythonwrench.serialization.csv import dump_csv, load_csv
|
|
415
|
+
|
|
416
|
+
if cache_store_mode != "outputs_only":
|
|
417
|
+
msg = f"Invalid combinaison of arguments {cache_saving_backend=} with {cache_store_mode=}."
|
|
418
|
+
raise ValueError(msg)
|
|
419
|
+
|
|
420
|
+
suffix = ".csv"
|
|
421
|
+
cache_dump_fn = dump_csv
|
|
422
|
+
cache_load_fn = load_csv
|
|
423
|
+
|
|
424
|
+
elif cache_saving_backend == "custom":
|
|
425
|
+
if cache_dump_fn is None or cache_load_fn is None:
|
|
426
|
+
msg = f"If {cache_saving_backend=}, arguments cache_dump_fn and cache_load_fn cannot be None. (found {cache_dump_fn=} {cache_load_fn=})"
|
|
427
|
+
raise ValueError(msg)
|
|
428
|
+
|
|
429
|
+
suffix = ""
|
|
430
|
+
else:
|
|
431
|
+
msg = f"Invalid argument {cache_saving_backend=}. (expected one of {get_args(SavingBackend)})"
|
|
432
|
+
raise ValueError(msg)
|
|
433
|
+
|
|
434
|
+
if isinstance(cache_fname_fmt, str):
|
|
435
|
+
cache_fname_fmt = cache_fname_fmt.format
|
|
436
|
+
|
|
437
|
+
def _disk_cache_impl_fn(fn: Callable[P, T]) -> Callable[P, T]:
|
|
438
|
+
"""Perform the disk cache impl fn operation."""
|
|
439
|
+
fn_fullname, fn_name = _get_fn_fullname_and_name(fn)
|
|
440
|
+
cache_fn_dpath = _get_fn_cache_dpath(fn, cache_dpath=cache_dpath)
|
|
441
|
+
|
|
442
|
+
if cache_force:
|
|
443
|
+
compute_start_msg = f"[{fn_name}] Force mode enabled, computing outputs'... (started at {{now}})"
|
|
444
|
+
else:
|
|
445
|
+
compute_start_msg = (
|
|
446
|
+
f"[{fn_name}] Cache missed, computing outputs... (started at {{now}})"
|
|
447
|
+
)
|
|
448
|
+
compute_end_msg = (
|
|
449
|
+
f"[{fn_name}] Outputs computed in {{duration:.1f}}s. (ended at {{now}})"
|
|
450
|
+
)
|
|
451
|
+
load_start_msg = f"[{fn_name}] Loading cache..."
|
|
452
|
+
load_end_msg = f"[{fn_name}] Cache loaded."
|
|
453
|
+
argnames = get_argnames(fn)
|
|
454
|
+
|
|
455
|
+
@wraps(fn)
|
|
456
|
+
def _disk_cache_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
|
|
457
|
+
"""Perform the disk cache wrapper operation."""
|
|
458
|
+
checksum_args = fn, args, kwargs
|
|
459
|
+
|
|
460
|
+
kwds = {}
|
|
461
|
+
|
|
462
|
+
if cache_fname_fmt_args is None or "fn_name" in cache_fname_fmt_args:
|
|
463
|
+
kwds["fn_name"] = fn_name
|
|
464
|
+
|
|
465
|
+
if cache_fname_fmt_args is None or "fn_fullname" in cache_fname_fmt_args:
|
|
466
|
+
kwds["fn_fullname"] = fn_fullname
|
|
467
|
+
|
|
468
|
+
if cache_fname_fmt_args is None or "suffix" in cache_fname_fmt_args:
|
|
469
|
+
kwds["suffix"] = suffix
|
|
470
|
+
|
|
471
|
+
if cache_fname_fmt_args is None or any(
|
|
472
|
+
k in cache_fname_fmt_args for k in ("csum", "checksum", "checksum_hex")
|
|
473
|
+
):
|
|
474
|
+
csum = cache_checksum_fn(checksum_args)
|
|
475
|
+
kwds["checksum"] = csum
|
|
476
|
+
kwds["checksum_hex"] = hex(csum)[2:]
|
|
477
|
+
else:
|
|
478
|
+
csum = None
|
|
479
|
+
|
|
480
|
+
inputs_kwds = {
|
|
481
|
+
argname: argval
|
|
482
|
+
for argname, argval in zip(argnames, args)
|
|
483
|
+
if cache_fname_fmt_args is None or argname in cache_fname_fmt_args
|
|
484
|
+
}
|
|
485
|
+
kwds.update(inputs_kwds)
|
|
486
|
+
kwds.update(kwargs)
|
|
487
|
+
|
|
488
|
+
cache_fname = cache_fname_fmt(**kwds)
|
|
489
|
+
cache_fpath = cache_fn_dpath.joinpath(cache_fname)
|
|
490
|
+
|
|
491
|
+
if not cache_enable:
|
|
492
|
+
output = fn(*args, **kwargs)
|
|
493
|
+
|
|
494
|
+
elif cache_force or not cache_fpath.exists():
|
|
495
|
+
if cache_verbose > 0:
|
|
496
|
+
logger.info(compute_start_msg.format(now=get_now()))
|
|
497
|
+
|
|
498
|
+
start = time.perf_counter()
|
|
499
|
+
output = fn(*args, **kwargs)
|
|
500
|
+
duration = time.perf_counter() - start
|
|
501
|
+
|
|
502
|
+
if cache_verbose > 0:
|
|
503
|
+
logger.info(
|
|
504
|
+
compute_end_msg.format(now=get_now(), duration=duration)
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
if cache_store_mode == "outputs_only":
|
|
508
|
+
cache_content = output
|
|
509
|
+
|
|
510
|
+
elif (
|
|
511
|
+
cache_store_mode == "outputs_metadata"
|
|
512
|
+
or cache_store_mode == "outputs_metadata_inputs"
|
|
513
|
+
):
|
|
514
|
+
input = (
|
|
515
|
+
(args, kwargs)
|
|
516
|
+
if cache_store_mode == "outputs_metadata_inputs"
|
|
517
|
+
else None
|
|
518
|
+
)
|
|
519
|
+
cache_content = {
|
|
520
|
+
"datetime": get_now(),
|
|
521
|
+
"duration": duration,
|
|
522
|
+
"checksum": csum,
|
|
523
|
+
"fn_fullname": fn_fullname,
|
|
524
|
+
"output": output,
|
|
525
|
+
"input": input,
|
|
526
|
+
}
|
|
527
|
+
else:
|
|
528
|
+
msg = f"Invalid argument {cache_store_mode=}. (expected one of {get_args(StoreMode)})"
|
|
529
|
+
raise ValueError(msg)
|
|
530
|
+
|
|
531
|
+
cache_fn_dpath.mkdir(parents=True, exist_ok=True)
|
|
532
|
+
cache_dump_fn(cache_content, cache_fpath) # type: ignore
|
|
533
|
+
|
|
534
|
+
elif cache_fpath.is_file():
|
|
535
|
+
if cache_verbose > 0:
|
|
536
|
+
logger.info(load_start_msg)
|
|
537
|
+
|
|
538
|
+
cache_content: Any = cache_load_fn(cache_fpath)
|
|
539
|
+
|
|
540
|
+
if cache_store_mode == "outputs_only":
|
|
541
|
+
output = cache_content
|
|
542
|
+
|
|
543
|
+
elif cache_store_mode == "outputs_metadata":
|
|
544
|
+
output = cache_content["output"]
|
|
545
|
+
|
|
546
|
+
elif cache_store_mode == "outputs_metadata_inputs":
|
|
547
|
+
output = cache_content["output"]
|
|
548
|
+
input_ = cache_content["input"]
|
|
549
|
+
if input_ is not None and input_ != (args, kwargs):
|
|
550
|
+
os.remove(cache_fpath)
|
|
551
|
+
return _disk_cache_wrapper(*args, **kwargs)
|
|
552
|
+
else:
|
|
553
|
+
msg = f"Invalid argument {cache_store_mode=}. (expected one of {get_args(StoreMode)})"
|
|
554
|
+
raise ValueError(msg)
|
|
555
|
+
|
|
556
|
+
if cache_verbose > 0:
|
|
557
|
+
logger.info(load_end_msg)
|
|
558
|
+
|
|
559
|
+
if cache_store_mode != "outputs_only" and cache_verbose > 1:
|
|
560
|
+
metadata = {k: v for k, v in cache_content.items() if k != "output"}
|
|
561
|
+
msgs = f"Found cache metadata:\n{metadata}".split("\n")
|
|
562
|
+
for msg in msgs:
|
|
563
|
+
logger.debug(msg)
|
|
564
|
+
|
|
565
|
+
else:
|
|
566
|
+
msg = f"Path {str(cache_fpath)} exists but it is not a file."
|
|
567
|
+
raise RuntimeError(msg)
|
|
568
|
+
|
|
569
|
+
return output
|
|
570
|
+
|
|
571
|
+
_disk_cache_wrapper.fn = fn # type: ignore
|
|
572
|
+
_disk_cache_wrapper.cache_fn_dpath = cache_fn_dpath # type: ignore
|
|
573
|
+
|
|
574
|
+
return _disk_cache_wrapper
|
|
575
|
+
|
|
576
|
+
return _disk_cache_impl_fn
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def get_cache_dpath(cache_dpath: Union[str, Path, None] = None) -> Path:
|
|
580
|
+
"""Returns defaults disk cache directory path, which is `~/.cache/disk_cache`."""
|
|
581
|
+
if cache_dpath is None:
|
|
582
|
+
cache_dpath = _DEFAULT_CACHE_DPATH
|
|
583
|
+
else:
|
|
584
|
+
cache_dpath = Path(cache_dpath)
|
|
585
|
+
return cache_dpath
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
def remove_fn_cache(
|
|
589
|
+
fn: Callable,
|
|
590
|
+
*,
|
|
591
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
592
|
+
) -> None:
|
|
593
|
+
"""Removes all caches for a specific function."""
|
|
594
|
+
cache_fn_dpath = _get_fn_cache_dpath(fn, cache_dpath=cache_dpath)
|
|
595
|
+
if cache_fn_dpath.is_dir():
|
|
596
|
+
shutil.rmtree(cache_fn_dpath)
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def _get_fn_cache_dpath(
|
|
600
|
+
fn: Callable,
|
|
601
|
+
*,
|
|
602
|
+
cache_dpath: Union[str, Path, None] = None,
|
|
603
|
+
) -> Path:
|
|
604
|
+
"""Perform the get fn cache dpath operation."""
|
|
605
|
+
_, fn_name = _get_fn_fullname_and_name(fn)
|
|
606
|
+
cache_dpath = get_cache_dpath(cache_dpath)
|
|
607
|
+
cache_fn_dpath = cache_dpath.joinpath(fn_name)
|
|
608
|
+
return cache_fn_dpath
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
def _get_fn_fullname_and_name(fn: Callable) -> Tuple[str, str]:
|
|
612
|
+
"""Perform the get fn fullname and name operation."""
|
|
613
|
+
fn_fullname = get_fullname(fn, inst_suffix="").replace("<locals>", "_locals_")
|
|
614
|
+
fn_name = fn_fullname.split(".")[-1]
|
|
615
|
+
return fn_fullname, fn_name
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
import logging
|
|
5
|
+
import platform
|
|
6
|
+
import sys
|
|
7
|
+
import warnings
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any, Dict
|
|
10
|
+
|
|
11
|
+
import pythonwrench
|
|
12
|
+
from pythonwrench.os import get_num_cpus_available
|
|
13
|
+
from pythonwrench.serialization.json import dumps_json
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def main_info() -> None:
|
|
19
|
+
"""Show main packages versions."""
|
|
20
|
+
with warnings.catch_warnings():
|
|
21
|
+
install_info = get_install_info()
|
|
22
|
+
dumped = dumps_json(install_info, to_builtins=True)
|
|
23
|
+
print(dumped)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def get_install_info() -> Dict[str, Any]:
|
|
27
|
+
"""Returns current installation information. Meant for debugging."""
|
|
28
|
+
return {
|
|
29
|
+
"os": platform.system(),
|
|
30
|
+
"architecture": platform.architecture()[0],
|
|
31
|
+
"num_cpus": get_num_cpus_available(),
|
|
32
|
+
"python": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
|
|
33
|
+
"pythonwrench": pythonwrench.__version__,
|
|
34
|
+
"pythonwrench_path": get_pythonwrench_repository_path(),
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def get_pythonwrench_repository_path() -> str:
|
|
39
|
+
"""Return the absolute path where the source code of this package is installed."""
|
|
40
|
+
return str(Path(__file__).parent.parent.parent)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__":
|
|
44
|
+
main_info()
|