pythonwrench 0.6.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. pythonwrench/__init__.py +490 -0
  2. pythonwrench/__main__.py +7 -0
  3. pythonwrench/_core.py +192 -0
  4. pythonwrench/abc.py +30 -0
  5. pythonwrench/argparse/__init__.py +81 -0
  6. pythonwrench/argparse/dataclass_.py +284 -0
  7. pythonwrench/argparse/parsers.py +619 -0
  8. pythonwrench/cast.py +247 -0
  9. pythonwrench/checksum.py +427 -0
  10. pythonwrench/collections/__init__.py +104 -0
  11. pythonwrench/collections/collections.py +900 -0
  12. pythonwrench/collections/prop.py +104 -0
  13. pythonwrench/collections/reducers.py +330 -0
  14. pythonwrench/concurrent.py +73 -0
  15. pythonwrench/csv.py +12 -0
  16. pythonwrench/dataclasses.py +117 -0
  17. pythonwrench/datetime.py +17 -0
  18. pythonwrench/difflib.py +39 -0
  19. pythonwrench/disk_cache.py +615 -0
  20. pythonwrench/entrypoints/info.py +44 -0
  21. pythonwrench/entrypoints/safe_rmdir.py +98 -0
  22. pythonwrench/entrypoints/tree.py +113 -0
  23. pythonwrench/enum.py +55 -0
  24. pythonwrench/functools.py +234 -0
  25. pythonwrench/hashlib.py +95 -0
  26. pythonwrench/importlib.py +243 -0
  27. pythonwrench/inspect.py +69 -0
  28. pythonwrench/json.py +12 -0
  29. pythonwrench/jsonl.py +12 -0
  30. pythonwrench/logging.py +252 -0
  31. pythonwrench/math.py +107 -0
  32. pythonwrench/os.py +226 -0
  33. pythonwrench/pickle.py +12 -0
  34. pythonwrench/random.py +60 -0
  35. pythonwrench/re.py +139 -0
  36. pythonwrench/semver.py +406 -0
  37. pythonwrench/serialization/__init__.py +70 -0
  38. pythonwrench/serialization/_core.py +70 -0
  39. pythonwrench/serialization/csv.py +493 -0
  40. pythonwrench/serialization/json.py +178 -0
  41. pythonwrench/serialization/jsonl.py +215 -0
  42. pythonwrench/serialization/pickle.py +186 -0
  43. pythonwrench/time.py +34 -0
  44. pythonwrench/typing/__init__.py +125 -0
  45. pythonwrench/typing/checks.py +551 -0
  46. pythonwrench/typing/classes.py +251 -0
  47. pythonwrench/warnings.py +118 -0
  48. pythonwrench-0.6.4.dist-info/METADATA +242 -0
  49. pythonwrench-0.6.4.dist-info/RECORD +52 -0
  50. pythonwrench-0.6.4.dist-info/WHEEL +4 -0
  51. pythonwrench-0.6.4.dist-info/entry_points.txt +10 -0
  52. pythonwrench-0.6.4.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,615 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+
4
+ import logging
5
+ import os
6
+ import shutil
7
+ import time
8
+ import warnings
9
+ from functools import wraps
10
+ from pathlib import Path
11
+ from typing import (
12
+ Any,
13
+ Callable,
14
+ Dict,
15
+ Iterable,
16
+ Literal,
17
+ Optional,
18
+ Tuple,
19
+ TypedDict,
20
+ TypeVar,
21
+ Union,
22
+ get_args,
23
+ overload,
24
+ )
25
+
26
+ from typing_extensions import ParamSpec
27
+
28
+ from pythonwrench.checksum import checksum_any
29
+ from pythonwrench.datetime import get_now
30
+ from pythonwrench.inspect import get_argnames, get_fullname
31
+
32
+ T = TypeVar("T")
33
+ P = ParamSpec("P")
34
+ U = TypeVar("U")
35
+
36
+
37
+ ChecksumFn = Callable[[Tuple[Callable[P, T], Tuple, Dict[str, Any]]], int]
38
+ SavingBackend = Literal["csv", "json", "pickle"]
39
+ StoreMode = Literal["outputs_only", "outputs_metadata", "outputs_metadata_inputs"]
40
+
41
+ _DEFAULT_CACHE_STORE_MODE: StoreMode = "outputs_only"
42
+
43
+
44
+ class _CacheMeta(TypedDict):
45
+ datetime: str
46
+ duration: float
47
+ checksum: int
48
+ fn_fullname: str
49
+ output: Any
50
+ input: Optional[Tuple[Any, Any]]
51
+
52
+
53
+ _DEFAULT_CACHE_DPATH = Path.home().joinpath(".cache", "disk_cache")
54
+
55
+
56
+ logger = logging.getLogger(__name__)
57
+
58
+
59
+ @overload
60
+ def disk_cache_decorator(
61
+ fn: None = None,
62
+ *,
63
+ cache_dpath: Union[str, Path, None] = None,
64
+ cache_force: bool = False,
65
+ cache_verbose: int = 0,
66
+ cache_checksum_fn: ChecksumFn = checksum_any,
67
+ cache_saving_backend: Literal["custom"],
68
+ cache_fname_fmt: Union[
69
+ str, Callable[..., str]
70
+ ] = "{fn_name}_{checksum_hex}{suffix}",
71
+ cache_fname_fmt_args: Optional[Iterable[str]] = None,
72
+ cache_dump_fn: Callable[[Any, Path], Any],
73
+ cache_load_fn: Callable[[Path], Any],
74
+ cache_enable: bool = True,
75
+ cache_store_mode: StoreMode,
76
+ ) -> Callable[[Callable[P, T]], Callable[P, T]]:
77
+ """Perform the disk cache decorator operation."""
78
+ ...
79
+
80
+
81
+ @overload
82
+ def disk_cache_decorator(
83
+ fn: None = None,
84
+ *,
85
+ cache_dpath: Union[str, Path, None] = None,
86
+ cache_force: bool = False,
87
+ cache_verbose: int = 0,
88
+ cache_checksum_fn: ChecksumFn = checksum_any,
89
+ cache_saving_backend: SavingBackend,
90
+ cache_fname_fmt: Union[
91
+ str, Callable[..., str]
92
+ ] = "{fn_name}_{checksum_hex}{suffix}",
93
+ cache_fname_fmt_args: Optional[Iterable[str]] = None,
94
+ cache_dump_fn: None = None,
95
+ cache_load_fn: None = None,
96
+ cache_enable: bool = True,
97
+ cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
98
+ ) -> Callable[[Callable[P, T]], Callable[P, T]]:
99
+ """Perform the disk cache decorator operation."""
100
+ ...
101
+
102
+
103
+ @overload
104
+ def disk_cache_decorator(
105
+ fn: None = None,
106
+ *,
107
+ cache_dpath: Union[str, Path, None] = None,
108
+ cache_force: bool = False,
109
+ cache_verbose: int = 0,
110
+ cache_checksum_fn: ChecksumFn = checksum_any,
111
+ cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto",
112
+ cache_fname_fmt: Union[
113
+ str, Callable[..., str]
114
+ ] = "{fn_name}_{checksum_hex}{suffix}",
115
+ cache_fname_fmt_args: Optional[Iterable[str]] = None,
116
+ cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None,
117
+ cache_load_fn: Optional[Callable[[Path], Any]] = None,
118
+ cache_enable: bool = True,
119
+ cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
120
+ ) -> Callable[[Callable[P, T]], Callable[P, T]]:
121
+ """Perform the disk cache decorator operation."""
122
+ ...
123
+
124
+
125
+ @overload
126
+ def disk_cache_decorator(
127
+ fn: Callable[P, T],
128
+ *,
129
+ cache_dpath: Union[str, Path, None] = None,
130
+ cache_force: bool = False,
131
+ cache_verbose: int = 0,
132
+ cache_checksum_fn: ChecksumFn = checksum_any,
133
+ cache_saving_backend: Literal["custom"],
134
+ cache_fname_fmt: Union[
135
+ str, Callable[..., str]
136
+ ] = "{fn_name}_{checksum_hex}{suffix}",
137
+ cache_fname_fmt_args: Optional[Iterable[str]] = None,
138
+ cache_dump_fn: Callable[[Any, Path], Any],
139
+ cache_load_fn: Callable[[Path], Any],
140
+ cache_enable: bool = True,
141
+ cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
142
+ ) -> Callable[P, T]:
143
+ """Perform the disk cache decorator operation."""
144
+ ...
145
+
146
+
147
+ @overload
148
+ def disk_cache_decorator(
149
+ fn: Callable[P, T],
150
+ *,
151
+ cache_dpath: Union[str, Path, None] = None,
152
+ cache_force: bool = False,
153
+ cache_verbose: int = 0,
154
+ cache_checksum_fn: ChecksumFn = checksum_any,
155
+ cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto",
156
+ cache_fname_fmt: Union[
157
+ str, Callable[..., str]
158
+ ] = "{fn_name}_{checksum_hex}{suffix}",
159
+ cache_fname_fmt_args: Optional[Iterable[str]] = None,
160
+ cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None,
161
+ cache_load_fn: Optional[Callable[[Path], Any]] = None,
162
+ cache_enable: bool = True,
163
+ cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
164
+ ) -> Callable[P, T]:
165
+ """Perform the disk cache decorator operation."""
166
+ ...
167
+
168
+
169
+ def disk_cache_decorator(
170
+ fn: Optional[Callable[P, T]] = None,
171
+ *,
172
+ cache_dpath: Union[str, Path, None] = None,
173
+ cache_force: bool = False,
174
+ cache_verbose: int = 0,
175
+ cache_checksum_fn: ChecksumFn = checksum_any,
176
+ cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto",
177
+ cache_fname_fmt: Union[
178
+ str, Callable[..., str]
179
+ ] = "{fn_name}_{checksum_hex}{suffix}",
180
+ cache_fname_fmt_args: Optional[Iterable[str]] = None,
181
+ cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None,
182
+ cache_load_fn: Optional[Callable[[Path], Any]] = None,
183
+ cache_enable: bool = True,
184
+ cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
185
+ ) -> Callable:
186
+ """Decorator to store function output in a cache file.
187
+
188
+ Cache file is identified by the checksum of the function arguments, and stored by default in `"~/.cache/disk_cache/<Function_name>/"` directory.
189
+
190
+ Example
191
+ -------
192
+ >>> import pythonwrench as pw
193
+ >>> @pw.disk_cache_decorator
194
+ >>> def heavy_processing():
195
+ >>> # Lot of stuff here
196
+ >>> ...
197
+ >>> outputs = heavy_processing() # first time function is called
198
+ >>> outputs = heavy_processing() # second time outputs is loaded from disk
199
+
200
+ Args:
201
+ fn: Function to store its output. By default, it must be a callable that returns a pickable object.
202
+ cache_dpath: Cache directory path. defaults to `"~/.cache/disk_cache"`.
203
+ cache_force: Force function call and overwrite cache. defaults to False.
204
+ cache_verbose: Set verbose logging level. Higher means more verbose. defaults to 0.
205
+ cache_checksum_fn: Checksum function to identify input arguments. defaults to ``pythonwrench.checksum_any``.
206
+ cache_saving_backend: Optional saving backend. Can be one of ('csv', 'json', 'pickle', 'custom', 'auto'). defaults to 'auto'.
207
+ cache_fname_fmt: Cache filename format. defaults to "{fn_name}_{checksum_hex}{suffix}".
208
+ cache_dump_fn: Dump/save function to store outputs and overwrite saving backend. defaults to None.
209
+ cache_load_fn: Load function to store outputs and overwrite saving backend. defaults to None.
210
+ cache_enable: Enable disk cache. If False, the function has no effect. defaults to True.
211
+ cache_store_mode: Disk cache storage mode. By default, it store function output and saved date into the cache file. defaults to 'outputs_metadata'.
212
+ """
213
+ impl_fn = _disk_cache_impl(
214
+ cache_dpath=cache_dpath,
215
+ cache_force=cache_force,
216
+ cache_verbose=cache_verbose,
217
+ cache_checksum_fn=cache_checksum_fn,
218
+ cache_saving_backend=cache_saving_backend,
219
+ cache_fname_fmt=cache_fname_fmt,
220
+ cache_fname_fmt_args=cache_fname_fmt_args,
221
+ cache_dump_fn=cache_dump_fn,
222
+ cache_load_fn=cache_load_fn,
223
+ cache_enable=cache_enable,
224
+ cache_store_mode=cache_store_mode,
225
+ )
226
+ if fn is not None:
227
+ return impl_fn(fn)
228
+ else:
229
+ return impl_fn
230
+
231
+
232
+ @overload
233
+ def disk_cache_call(
234
+ fn: Callable[..., T],
235
+ *args,
236
+ cache_dpath: Union[str, Path, None] = None,
237
+ cache_force: bool = False,
238
+ cache_verbose: int = 0,
239
+ cache_checksum_fn: ChecksumFn = checksum_any,
240
+ cache_saving_backend: Literal["custom"],
241
+ cache_fname_fmt: Union[
242
+ str, Callable[..., str]
243
+ ] = "{fn_name}_{checksum_hex}{suffix}",
244
+ cache_fname_fmt_args: Optional[Iterable[str]] = None,
245
+ cache_dump_fn: Callable[[Any, Path], Any],
246
+ cache_load_fn: Callable[[Path], Any],
247
+ cache_enable: bool = True,
248
+ cache_store_mode: StoreMode,
249
+ **kwargs,
250
+ ) -> T:
251
+ """Perform the disk cache call operation."""
252
+ ...
253
+
254
+
255
+ @overload
256
+ def disk_cache_call(
257
+ fn: Callable[..., T],
258
+ *args,
259
+ cache_dpath: Union[str, Path, None] = None,
260
+ cache_force: bool = False,
261
+ cache_verbose: int = 0,
262
+ cache_checksum_fn: ChecksumFn = checksum_any,
263
+ cache_saving_backend: SavingBackend,
264
+ cache_fname_fmt: Union[
265
+ str, Callable[..., str]
266
+ ] = "{fn_name}_{checksum_hex}{suffix}",
267
+ cache_fname_fmt_args: Optional[Iterable[str]] = None,
268
+ cache_dump_fn: None = None,
269
+ cache_load_fn: None = None,
270
+ cache_enable: bool = True,
271
+ cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
272
+ **kwargs,
273
+ ) -> T:
274
+ """Perform the disk cache call operation."""
275
+ ...
276
+
277
+
278
+ @overload
279
+ def disk_cache_call(
280
+ fn: Callable[..., T],
281
+ *args,
282
+ cache_dpath: Union[str, Path, None] = None,
283
+ cache_force: bool = False,
284
+ cache_verbose: int = 0,
285
+ cache_checksum_fn: ChecksumFn = checksum_any,
286
+ cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto",
287
+ cache_fname_fmt: Union[
288
+ str, Callable[..., str]
289
+ ] = "{fn_name}_{checksum_hex}{suffix}",
290
+ cache_fname_fmt_args: Optional[Iterable[str]] = None,
291
+ cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None,
292
+ cache_load_fn: Optional[Callable[[Path], Any]] = None,
293
+ cache_enable: bool = True,
294
+ cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
295
+ **kwargs,
296
+ ) -> T:
297
+ """Perform the disk cache call operation."""
298
+ ...
299
+
300
+
301
+ def disk_cache_call(
302
+ fn: Callable[..., T],
303
+ *args,
304
+ cache_dpath: Union[str, Path, None] = None,
305
+ cache_force: bool = False,
306
+ cache_verbose: int = 0,
307
+ cache_checksum_fn: ChecksumFn = checksum_any,
308
+ cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto",
309
+ cache_fname_fmt: Union[
310
+ str, Callable[..., str]
311
+ ] = "{fn_name}_{checksum_hex}{suffix}",
312
+ cache_fname_fmt_args: Optional[Iterable[str]] = None,
313
+ cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None,
314
+ cache_load_fn: Optional[Callable[[Path], Any]] = None,
315
+ cache_enable: bool = True,
316
+ cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
317
+ **kwargs,
318
+ ) -> T:
319
+ r"""Call function and store output in a cache file.
320
+
321
+ Cache file is identified by the checksum of the function arguments, and stored by default in '~/.cache/disk_cache/<Function_name>/' directory.
322
+
323
+ Example
324
+ -------
325
+ >>> import pythonwrench as pw
326
+ >>> def heavy_processing():
327
+ >>> # Lot of stuff here
328
+ >>> ...
329
+ >>> outputs = pw.disk_cache_call(heavy_processing) # first time function is called
330
+ >>> outputs = pw.disk_cache_call(heavy_processing) # second time outputs is loaded from disk
331
+
332
+ Args:
333
+ fn: Function to store its output. By default, it must be a callable that returns a pickable object.
334
+ cache_dpath: Cache directory path. defaults to '~/.cache/disk_cache'.
335
+ cache_force: Force function call and overwrite cache. defaults to False.
336
+ cache_verbose: Set verbose logging level. Higher means more verbose. defaults to 0.
337
+ cache_checksum_fn: Checksum function to identify input arguments. defaults to ``pythonwrench.checksum_any``.
338
+ cache_saving_backend: Optional saving backend. Can be one of ('csv', 'json', 'pickle', 'custom', 'auto'). defaults to 'auto'.
339
+ cache_fname_fmt: Cache filename format. defaults to '{fn_name}_{checksum_hex}{suffix}'.
340
+ cache_dump_fn: Dump/save function to store outputs and overwrite saving backend. defaults to None.
341
+ cache_load_fn: Load function to store outputs and overwrite saving backend. defaults to None.
342
+ cache_enable: Enable disk cache. If False, the function has no effect. defaults to True.
343
+ cache_store_mode: Disk cache storage mode. By default, it store function output and saved date into the cache file. defaults to 'outputs_metadata'.
344
+ \*args: Positional arguments passed to the function.
345
+ \*\*kwargs: Keywords arguments passed to the function.
346
+ """
347
+ wrapped_fn = _disk_cache_impl(
348
+ cache_dpath=cache_dpath,
349
+ cache_force=cache_force,
350
+ cache_verbose=cache_verbose,
351
+ cache_checksum_fn=cache_checksum_fn,
352
+ cache_saving_backend=cache_saving_backend,
353
+ cache_fname_fmt=cache_fname_fmt,
354
+ cache_fname_fmt_args=cache_fname_fmt_args,
355
+ cache_dump_fn=cache_dump_fn,
356
+ cache_load_fn=cache_load_fn,
357
+ cache_enable=cache_enable,
358
+ cache_store_mode=cache_store_mode,
359
+ )
360
+ return wrapped_fn(fn)(*args, **kwargs)
361
+
362
+
363
+ def _disk_cache_impl(
364
+ *,
365
+ cache_dpath: Union[str, Path, None] = None,
366
+ cache_force: bool = False,
367
+ cache_verbose: int = 0,
368
+ cache_checksum_fn: ChecksumFn = checksum_any,
369
+ cache_saving_backend: Union[SavingBackend, Literal["custom", "auto"]] = "auto",
370
+ cache_fname_fmt: Union[
371
+ str, Callable[..., str]
372
+ ] = "{fn_name}_{checksum_hex}{suffix}",
373
+ cache_fname_fmt_args: Optional[Iterable[str]] = None,
374
+ cache_dump_fn: Optional[Callable[[Any, Path], Any]] = None,
375
+ cache_load_fn: Optional[Callable[[Path], Any]] = None,
376
+ cache_enable: bool = True,
377
+ cache_store_mode: StoreMode = _DEFAULT_CACHE_STORE_MODE,
378
+ ) -> Callable[[Callable[P, T]], Callable[P, T]]:
379
+ # for backward compatibility
380
+ """Perform the disk cache impl operation."""
381
+ if cache_fname_fmt is None:
382
+ expected = "{fn_name}_{csum}{suffix}"
383
+ msg = f"Deprecated argument value {cache_fname_fmt=}. (use {expected} instead)"
384
+ warnings.warn(msg, DeprecationWarning)
385
+ cache_fname_fmt = expected
386
+
387
+ if cache_saving_backend is None:
388
+ expected = "auto"
389
+ msg = f"Deprecated argument value {cache_saving_backend=}. (use {expected} instead)"
390
+ warnings.warn(msg, DeprecationWarning)
391
+ cache_saving_backend = expected
392
+
393
+ if cache_saving_backend == "auto":
394
+ if cache_dump_fn is not None and cache_load_fn is not None:
395
+ cache_saving_backend = "custom"
396
+ else:
397
+ cache_saving_backend = "pickle"
398
+
399
+ if cache_saving_backend == "pickle":
400
+ from pythonwrench.serialization.pickle import dump_pickle, load_pickle
401
+
402
+ suffix = ".pickle"
403
+ cache_dump_fn = dump_pickle
404
+ cache_load_fn = load_pickle
405
+
406
+ elif cache_saving_backend == "json":
407
+ from pythonwrench.serialization.json import dump_json, load_json
408
+
409
+ suffix = ".json"
410
+ cache_dump_fn = dump_json
411
+ cache_load_fn = load_json
412
+
413
+ elif cache_saving_backend == "csv":
414
+ from pythonwrench.serialization.csv import dump_csv, load_csv
415
+
416
+ if cache_store_mode != "outputs_only":
417
+ msg = f"Invalid combinaison of arguments {cache_saving_backend=} with {cache_store_mode=}."
418
+ raise ValueError(msg)
419
+
420
+ suffix = ".csv"
421
+ cache_dump_fn = dump_csv
422
+ cache_load_fn = load_csv
423
+
424
+ elif cache_saving_backend == "custom":
425
+ if cache_dump_fn is None or cache_load_fn is None:
426
+ msg = f"If {cache_saving_backend=}, arguments cache_dump_fn and cache_load_fn cannot be None. (found {cache_dump_fn=} {cache_load_fn=})"
427
+ raise ValueError(msg)
428
+
429
+ suffix = ""
430
+ else:
431
+ msg = f"Invalid argument {cache_saving_backend=}. (expected one of {get_args(SavingBackend)})"
432
+ raise ValueError(msg)
433
+
434
+ if isinstance(cache_fname_fmt, str):
435
+ cache_fname_fmt = cache_fname_fmt.format
436
+
437
+ def _disk_cache_impl_fn(fn: Callable[P, T]) -> Callable[P, T]:
438
+ """Perform the disk cache impl fn operation."""
439
+ fn_fullname, fn_name = _get_fn_fullname_and_name(fn)
440
+ cache_fn_dpath = _get_fn_cache_dpath(fn, cache_dpath=cache_dpath)
441
+
442
+ if cache_force:
443
+ compute_start_msg = f"[{fn_name}] Force mode enabled, computing outputs'... (started at {{now}})"
444
+ else:
445
+ compute_start_msg = (
446
+ f"[{fn_name}] Cache missed, computing outputs... (started at {{now}})"
447
+ )
448
+ compute_end_msg = (
449
+ f"[{fn_name}] Outputs computed in {{duration:.1f}}s. (ended at {{now}})"
450
+ )
451
+ load_start_msg = f"[{fn_name}] Loading cache..."
452
+ load_end_msg = f"[{fn_name}] Cache loaded."
453
+ argnames = get_argnames(fn)
454
+
455
+ @wraps(fn)
456
+ def _disk_cache_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
457
+ """Perform the disk cache wrapper operation."""
458
+ checksum_args = fn, args, kwargs
459
+
460
+ kwds = {}
461
+
462
+ if cache_fname_fmt_args is None or "fn_name" in cache_fname_fmt_args:
463
+ kwds["fn_name"] = fn_name
464
+
465
+ if cache_fname_fmt_args is None or "fn_fullname" in cache_fname_fmt_args:
466
+ kwds["fn_fullname"] = fn_fullname
467
+
468
+ if cache_fname_fmt_args is None or "suffix" in cache_fname_fmt_args:
469
+ kwds["suffix"] = suffix
470
+
471
+ if cache_fname_fmt_args is None or any(
472
+ k in cache_fname_fmt_args for k in ("csum", "checksum", "checksum_hex")
473
+ ):
474
+ csum = cache_checksum_fn(checksum_args)
475
+ kwds["checksum"] = csum
476
+ kwds["checksum_hex"] = hex(csum)[2:]
477
+ else:
478
+ csum = None
479
+
480
+ inputs_kwds = {
481
+ argname: argval
482
+ for argname, argval in zip(argnames, args)
483
+ if cache_fname_fmt_args is None or argname in cache_fname_fmt_args
484
+ }
485
+ kwds.update(inputs_kwds)
486
+ kwds.update(kwargs)
487
+
488
+ cache_fname = cache_fname_fmt(**kwds)
489
+ cache_fpath = cache_fn_dpath.joinpath(cache_fname)
490
+
491
+ if not cache_enable:
492
+ output = fn(*args, **kwargs)
493
+
494
+ elif cache_force or not cache_fpath.exists():
495
+ if cache_verbose > 0:
496
+ logger.info(compute_start_msg.format(now=get_now()))
497
+
498
+ start = time.perf_counter()
499
+ output = fn(*args, **kwargs)
500
+ duration = time.perf_counter() - start
501
+
502
+ if cache_verbose > 0:
503
+ logger.info(
504
+ compute_end_msg.format(now=get_now(), duration=duration)
505
+ )
506
+
507
+ if cache_store_mode == "outputs_only":
508
+ cache_content = output
509
+
510
+ elif (
511
+ cache_store_mode == "outputs_metadata"
512
+ or cache_store_mode == "outputs_metadata_inputs"
513
+ ):
514
+ input = (
515
+ (args, kwargs)
516
+ if cache_store_mode == "outputs_metadata_inputs"
517
+ else None
518
+ )
519
+ cache_content = {
520
+ "datetime": get_now(),
521
+ "duration": duration,
522
+ "checksum": csum,
523
+ "fn_fullname": fn_fullname,
524
+ "output": output,
525
+ "input": input,
526
+ }
527
+ else:
528
+ msg = f"Invalid argument {cache_store_mode=}. (expected one of {get_args(StoreMode)})"
529
+ raise ValueError(msg)
530
+
531
+ cache_fn_dpath.mkdir(parents=True, exist_ok=True)
532
+ cache_dump_fn(cache_content, cache_fpath) # type: ignore
533
+
534
+ elif cache_fpath.is_file():
535
+ if cache_verbose > 0:
536
+ logger.info(load_start_msg)
537
+
538
+ cache_content: Any = cache_load_fn(cache_fpath)
539
+
540
+ if cache_store_mode == "outputs_only":
541
+ output = cache_content
542
+
543
+ elif cache_store_mode == "outputs_metadata":
544
+ output = cache_content["output"]
545
+
546
+ elif cache_store_mode == "outputs_metadata_inputs":
547
+ output = cache_content["output"]
548
+ input_ = cache_content["input"]
549
+ if input_ is not None and input_ != (args, kwargs):
550
+ os.remove(cache_fpath)
551
+ return _disk_cache_wrapper(*args, **kwargs)
552
+ else:
553
+ msg = f"Invalid argument {cache_store_mode=}. (expected one of {get_args(StoreMode)})"
554
+ raise ValueError(msg)
555
+
556
+ if cache_verbose > 0:
557
+ logger.info(load_end_msg)
558
+
559
+ if cache_store_mode != "outputs_only" and cache_verbose > 1:
560
+ metadata = {k: v for k, v in cache_content.items() if k != "output"}
561
+ msgs = f"Found cache metadata:\n{metadata}".split("\n")
562
+ for msg in msgs:
563
+ logger.debug(msg)
564
+
565
+ else:
566
+ msg = f"Path {str(cache_fpath)} exists but it is not a file."
567
+ raise RuntimeError(msg)
568
+
569
+ return output
570
+
571
+ _disk_cache_wrapper.fn = fn # type: ignore
572
+ _disk_cache_wrapper.cache_fn_dpath = cache_fn_dpath # type: ignore
573
+
574
+ return _disk_cache_wrapper
575
+
576
+ return _disk_cache_impl_fn
577
+
578
+
579
+ def get_cache_dpath(cache_dpath: Union[str, Path, None] = None) -> Path:
580
+ """Returns defaults disk cache directory path, which is `~/.cache/disk_cache`."""
581
+ if cache_dpath is None:
582
+ cache_dpath = _DEFAULT_CACHE_DPATH
583
+ else:
584
+ cache_dpath = Path(cache_dpath)
585
+ return cache_dpath
586
+
587
+
588
+ def remove_fn_cache(
589
+ fn: Callable,
590
+ *,
591
+ cache_dpath: Union[str, Path, None] = None,
592
+ ) -> None:
593
+ """Removes all caches for a specific function."""
594
+ cache_fn_dpath = _get_fn_cache_dpath(fn, cache_dpath=cache_dpath)
595
+ if cache_fn_dpath.is_dir():
596
+ shutil.rmtree(cache_fn_dpath)
597
+
598
+
599
+ def _get_fn_cache_dpath(
600
+ fn: Callable,
601
+ *,
602
+ cache_dpath: Union[str, Path, None] = None,
603
+ ) -> Path:
604
+ """Perform the get fn cache dpath operation."""
605
+ _, fn_name = _get_fn_fullname_and_name(fn)
606
+ cache_dpath = get_cache_dpath(cache_dpath)
607
+ cache_fn_dpath = cache_dpath.joinpath(fn_name)
608
+ return cache_fn_dpath
609
+
610
+
611
+ def _get_fn_fullname_and_name(fn: Callable) -> Tuple[str, str]:
612
+ """Perform the get fn fullname and name operation."""
613
+ fn_fullname = get_fullname(fn, inst_suffix="").replace("<locals>", "_locals_")
614
+ fn_name = fn_fullname.split(".")[-1]
615
+ return fn_fullname, fn_name
@@ -0,0 +1,44 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: utf-8 -*-
3
+
4
+ import logging
5
+ import platform
6
+ import sys
7
+ import warnings
8
+ from pathlib import Path
9
+ from typing import Any, Dict
10
+
11
+ import pythonwrench
12
+ from pythonwrench.os import get_num_cpus_available
13
+ from pythonwrench.serialization.json import dumps_json
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+
18
+ def main_info() -> None:
19
+ """Show main packages versions."""
20
+ with warnings.catch_warnings():
21
+ install_info = get_install_info()
22
+ dumped = dumps_json(install_info, to_builtins=True)
23
+ print(dumped)
24
+
25
+
26
+ def get_install_info() -> Dict[str, Any]:
27
+ """Returns current installation information. Meant for debugging."""
28
+ return {
29
+ "os": platform.system(),
30
+ "architecture": platform.architecture()[0],
31
+ "num_cpus": get_num_cpus_available(),
32
+ "python": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
33
+ "pythonwrench": pythonwrench.__version__,
34
+ "pythonwrench_path": get_pythonwrench_repository_path(),
35
+ }
36
+
37
+
38
+ def get_pythonwrench_repository_path() -> str:
39
+ """Return the absolute path where the source code of this package is installed."""
40
+ return str(Path(__file__).parent.parent.parent)
41
+
42
+
43
+ if __name__ == "__main__":
44
+ main_info()