diskcache-community 5.8.0__tar.gz → 5.8.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. {diskcache_community-5.8.0/diskcache_community.egg-info → diskcache_community-5.8.2}/PKG-INFO +4 -3
  2. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/README.md +2 -1
  3. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/diskcache_community/__init__.py +4 -2
  4. diskcache_community-5.8.2/diskcache_community/__init__.pyi +60 -0
  5. diskcache_community-5.8.2/diskcache_community/_types.pyi +227 -0
  6. diskcache_community-5.8.2/diskcache_community/arrays.py +322 -0
  7. diskcache_community-5.8.2/diskcache_community/arrays.pyi +8 -0
  8. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/diskcache_community/core.py +379 -96
  9. diskcache_community-5.8.2/diskcache_community/core.pyi +335 -0
  10. diskcache_community-5.8.2/diskcache_community/djangocache.pyi +101 -0
  11. diskcache_community-5.8.2/diskcache_community/fanout.pyi +70 -0
  12. diskcache_community-5.8.2/diskcache_community/persistent.pyi +161 -0
  13. diskcache_community-5.8.2/diskcache_community/py.typed +0 -0
  14. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/diskcache_community/recipes.py +1 -1
  15. diskcache_community-5.8.2/diskcache_community/recipes.pyi +102 -0
  16. {diskcache_community-5.8.0 → diskcache_community-5.8.2/diskcache_community.egg-info}/PKG-INFO +4 -3
  17. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/diskcache_community.egg-info/SOURCES.txt +13 -1
  18. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/pyproject.toml +1 -1
  19. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_core.py +116 -5
  20. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_default_disk.py +20 -0
  21. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_entry_errors.py +2 -0
  22. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_golden.py +7 -0
  23. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_readonly.py +10 -0
  24. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_recipes.py +20 -0
  25. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_safedisk.py +15 -0
  26. diskcache_community-5.8.2/tests/test_safedisk_numpy.py +1403 -0
  27. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_safedisk_types.py +35 -0
  28. diskcache_community-5.8.2/tests/test_safejsondisk.py +390 -0
  29. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/LICENSE +0 -0
  30. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/NOTICE +0 -0
  31. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/diskcache_community/cli.py +0 -0
  32. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/diskcache_community/djangocache.py +0 -0
  33. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/diskcache_community/fanout.py +0 -0
  34. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/diskcache_community/persistent.py +0 -0
  35. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/diskcache_community.egg-info/dependency_links.txt +0 -0
  36. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/diskcache_community.egg-info/top_level.txt +0 -0
  37. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/setup.cfg +0 -0
  38. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_deque.py +0 -0
  39. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_djangocache.py +0 -0
  40. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_doctest.py +0 -0
  41. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_fanout.py +0 -0
  42. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_index.py +0 -0
  43. {diskcache_community-5.8.0 → diskcache_community-5.8.2}/tests/test_init.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: diskcache-community
3
- Version: 5.8.0
3
+ Version: 5.8.2
4
4
  Summary: Disk Cache -- Disk and file backed persistent cache.
5
5
  Author: Grant Jenks
6
6
  Maintainer: diskcache-community contributors
@@ -22,7 +22,7 @@ Classifier: Programming Language :: Python :: 3.12
22
22
  Classifier: Programming Language :: Python :: 3.13
23
23
  Classifier: Programming Language :: Python :: 3.14
24
24
  Classifier: Programming Language :: Python :: Implementation :: CPython
25
- Requires-Python: >=3.10
25
+ Requires-Python: >=3.8
26
26
  Description-Content-Type: text/markdown
27
27
  License-File: LICENSE
28
28
  License-File: NOTICE
@@ -142,7 +142,8 @@ cache.
142
142
  semaphores and throttling
143
143
  - Thread-safe and process-safe, with fork-safe connection handling
144
144
  - Read-only mode, automatic when the directory is not writable
145
- - Opt-in `SafeDisk` for caches whose directory other users can write to
145
+ - Opt-in `SafeDisk` and `SafeJSONDisk` for caches whose directory other
146
+ users can write to, with `SafeDisk.with_numpy()` for numpy arrays
146
147
  - Pure Python on top of the standard library `sqlite3`; no server, no compiler
147
148
  - Tested on CPython 3.10 to 3.14 on Linux, macOS and Windows
148
149
  - Gets in about 2.6 µs and sets in about 19 µs on an Apple M5 Pro in the
@@ -112,7 +112,8 @@ cache.
112
112
  semaphores and throttling
113
113
  - Thread-safe and process-safe, with fork-safe connection handling
114
114
  - Read-only mode, automatic when the directory is not writable
115
- - Opt-in `SafeDisk` for caches whose directory other users can write to
115
+ - Opt-in `SafeDisk` and `SafeJSONDisk` for caches whose directory other
116
+ users can write to, with `SafeDisk.with_numpy()` for numpy arrays
116
117
  - Pure Python on top of the standard library `sqlite3`; no server, no compiler
117
118
  - Tested on CPython 3.10 to 3.14 on Linux, macOS and Windows
118
119
  - Gets in about 2.6 µs and sets in about 19 µs on an Apple M5 Pro in the
@@ -16,6 +16,7 @@ from .core import (
16
16
  JSONDisk,
17
17
  ReadOnlyError,
18
18
  SafeDisk,
19
+ SafeJSONDisk,
19
20
  Timeout,
20
21
  UnknownFileWarning,
21
22
  configure,
@@ -50,6 +51,7 @@ __all__ = [
50
51
  'RLock',
51
52
  'ReadOnlyError',
52
53
  'SafeDisk',
54
+ 'SafeJSONDisk',
53
55
  'Timeout',
54
56
  'UNKNOWN',
55
57
  'UnknownFileWarning',
@@ -70,8 +72,8 @@ except Exception: # pylint: disable=broad-except # pragma: no cover
70
72
 
71
73
 
72
74
  __title__ = 'diskcache-community'
73
- __version__ = '5.8.0'
74
- __build__ = 0x050800
75
+ __version__ = '5.8.2'
76
+ __build__ = 0x050802
75
77
  __author__ = 'Grant Jenks'
76
78
  __license__ = 'Apache 2.0'
77
79
  __copyright__ = 'Copyright 2016-2023 Grant Jenks'
@@ -0,0 +1,60 @@
1
+ from diskcache_community.core import DEFAULT_SETTINGS as DEFAULT_SETTINGS
2
+ from diskcache_community.core import ENOVAL as ENOVAL
3
+ from diskcache_community.core import EVICTION_POLICY as EVICTION_POLICY
4
+ from diskcache_community.core import UNKNOWN as UNKNOWN
5
+ from diskcache_community.core import Cache as Cache
6
+ from diskcache_community.core import Disk as Disk
7
+ from diskcache_community.core import EmptyDirWarning as EmptyDirWarning
8
+ from diskcache_community.core import JSONDisk as JSONDisk
9
+ from diskcache_community.core import ReadOnlyError as ReadOnlyError
10
+ from diskcache_community.core import SafeDisk as SafeDisk
11
+ from diskcache_community.core import SafeJSONDisk as SafeJSONDisk
12
+ from diskcache_community.core import Timeout as Timeout
13
+ from diskcache_community.core import UnknownFileWarning as UnknownFileWarning
14
+ from diskcache_community.core import configure as configure
15
+ from diskcache_community.core import default_disk as default_disk
16
+ from diskcache_community.fanout import FanoutCache as FanoutCache
17
+ from diskcache_community.persistent import Deque as Deque
18
+ from diskcache_community.persistent import Index as Index
19
+ from diskcache_community.recipes import Averager as Averager
20
+ from diskcache_community.recipes import BoundedSemaphore as BoundedSemaphore
21
+ from diskcache_community.recipes import Lock as Lock
22
+ from diskcache_community.recipes import RLock as RLock
23
+ from diskcache_community.recipes import barrier as barrier
24
+ from diskcache_community.recipes import memoize_stampede as memoize_stampede
25
+ from diskcache_community.recipes import throttle as throttle
26
+
27
+ __all__ = [
28
+ 'Averager',
29
+ 'BoundedSemaphore',
30
+ 'Cache',
31
+ 'DEFAULT_SETTINGS',
32
+ 'Deque',
33
+ 'Disk',
34
+ 'ENOVAL',
35
+ 'EVICTION_POLICY',
36
+ 'EmptyDirWarning',
37
+ 'FanoutCache',
38
+ 'Index',
39
+ 'JSONDisk',
40
+ 'Lock',
41
+ 'RLock',
42
+ 'ReadOnlyError',
43
+ 'SafeDisk',
44
+ 'SafeJSONDisk',
45
+ 'Timeout',
46
+ 'UNKNOWN',
47
+ 'UnknownFileWarning',
48
+ 'barrier',
49
+ 'configure',
50
+ 'default_disk',
51
+ 'memoize_stampede',
52
+ 'throttle',
53
+ ]
54
+
55
+ __title__: str
56
+ __version__: str
57
+ __build__: int
58
+ __author__: str
59
+ __license__: str
60
+ __copyright__: str
@@ -0,0 +1,227 @@
1
+ import warnings
2
+ from typing import (
3
+ Any,
4
+ Callable,
5
+ Container,
6
+ ContextManager,
7
+ Iterator,
8
+ Literal,
9
+ Protocol,
10
+ overload,
11
+ )
12
+
13
+ from _typeshed import Incomplete
14
+ from typing_extensions import ParamSpec, Self, TypeAlias, TypedDict, TypeVar
15
+
16
+ from diskcache_community import Disk
17
+
18
+ _T = TypeVar('_T')
19
+ _IntOrStrT = TypeVar('_IntOrStrT', bound=int | str)
20
+ _T_co = TypeVar('_T_co', covariant=True)
21
+ _P = ParamSpec('_P')
22
+
23
+ DbName: TypeAlias = Literal['cache.db']
24
+ ModeNone: TypeAlias = Literal[0]
25
+ ModeRaw: TypeAlias = Literal[1]
26
+ ModeBinary: TypeAlias = Literal[2]
27
+ ModeText: TypeAlias = Literal[3]
28
+ ModePickle: TypeAlias = Literal[4]
29
+ ModeLiteral: TypeAlias = (
30
+ ModeNone | ModeRaw | ModeBinary | ModeText | ModePickle
31
+ )
32
+ EnovalType: TypeAlias = Literal['ENOVAL']
33
+ UnknownType: TypeAlias = Literal['UNKNOWN']
34
+
35
+ ExpireTime: TypeAlias = float | None
36
+ Tag: TypeAlias = str | None
37
+ Ignore: TypeAlias = Container[int | str]
38
+
39
+ ServerSide: TypeAlias = Literal['back', 'front']
40
+ KeyValuePair: TypeAlias = tuple[_IntOrStrT, Any]
41
+ NullPair: TypeAlias = tuple[None, None]
42
+ NullablePair: TypeAlias = KeyValuePair[_IntOrStrT] | NullPair
43
+
44
+ EvictionPolicyKey: TypeAlias = Literal[
45
+ 'none',
46
+ 'least-recently-stored',
47
+ 'least-recently-used',
48
+ 'least-frequently-used',
49
+ ]
50
+ EvictionPolicyItemKey: TypeAlias = Literal['init', 'get', 'cull']
51
+ EvictionPolicyItem: TypeAlias = dict[EvictionPolicyItemKey, str | None]
52
+
53
+ class Memoized(Protocol[_P, _T_co]):
54
+ def __cache_key__(
55
+ self, *args: _P.args, **kwargs: _P.kwargs
56
+ ) -> tuple[Any, ...]: ...
57
+ def __call__(self, *args: _P.args, **kwargs: _P.kwargs) -> _T_co: ...
58
+
59
+ class Settings(TypedDict, total=False):
60
+ statistics: int
61
+ tag_index: int
62
+ eviction_policy: str
63
+ size_limit: int
64
+ cull_limit: int
65
+ sqlite_auto_vacuum: int
66
+ sqlite_cache_size: int
67
+ sqlite_journal_mode: str
68
+ sqlite_mmap_size: int
69
+ sqlite_synchronous: int
70
+ disk_min_file_size: int
71
+ disk_pickle_protocol: int
72
+
73
+ class DefaultSettings(TypedDict, total=True):
74
+ statistics: int
75
+ tag_index: int
76
+ eviction_policy: str
77
+ size_limit: int
78
+ cull_limit: int
79
+ sqlite_auto_vacuum: int
80
+ sqlite_cache_size: int
81
+ sqlite_journal_mode: str
82
+ sqlite_mmap_size: int
83
+ sqlite_synchronous: int
84
+ disk_min_file_size: int
85
+ disk_pickle_protocol: int
86
+
87
+ class Metadata(TypedDict, total=False):
88
+ count: int
89
+ size: int
90
+ hits: int
91
+ misses: int
92
+
93
+ class DefaultMetadata(TypedDict, total=True):
94
+ count: int
95
+ size: int
96
+ hits: int
97
+ misses: int
98
+
99
+ class InitSettings(TypedDict, total=False):
100
+ # settings
101
+ statistics: int
102
+ tag_index: int
103
+ eviction_policy: str
104
+ size_limit: int
105
+ cull_limit: int
106
+ sqlite_auto_vacuum: int
107
+ sqlite_cache_size: int
108
+ sqlite_journal_mode: str
109
+ sqlite_mmap_size: int
110
+ sqlite_synchronous: int
111
+ disk_min_file_size: int
112
+ disk_pickle_protocol: int
113
+ # metadata
114
+ count: int
115
+ size: int
116
+ hits: int
117
+ misses: int
118
+
119
+ class BaseCache(Protocol):
120
+ ### Settings
121
+ @property
122
+ def statistics(self) -> int: ...
123
+ @property
124
+ def tag_index(self) -> int: ...
125
+ @property
126
+ def eviction_policy(self) -> str: ...
127
+ @property
128
+ def size_limit(self) -> int: ...
129
+ @property
130
+ def cull_limit(self) -> int: ...
131
+ @property
132
+ def sqlite_auto_vacuum(self) -> int: ...
133
+ @property
134
+ def sqlite_cache_size(self) -> int: ...
135
+ @property
136
+ def sqlite_journal_mode(self) -> str: ...
137
+ @property
138
+ def sqlite_mmap_size(self) -> int: ...
139
+ @property
140
+ def sqlite_synchronous(self) -> int: ...
141
+ @property
142
+ def disk_min_file_size(self) -> int: ...
143
+ @property
144
+ def disk_pickle_protocol(self) -> int: ...
145
+ ###
146
+ @property
147
+ def directory(self) -> str: ...
148
+ @property
149
+ def timeout(self) -> int: ...
150
+ @property
151
+ def disk(self) -> Disk: ...
152
+ def transact(self, retry: bool = ...) -> ContextManager[None]: ...
153
+ def set( # noqa: PLR0913
154
+ self,
155
+ key: Any,
156
+ value: Any,
157
+ expire: ExpireTime = ...,
158
+ read: bool = ...,
159
+ tag: Tag = ...,
160
+ retry: bool = ...,
161
+ ) -> Literal[True]: ...
162
+ def __setitem__(self, key: Any, value: Any) -> None: ...
163
+ def touch(
164
+ self, key: Any, expire: ExpireTime = ..., retry: bool = ...
165
+ ) -> bool: ...
166
+ def add( # noqa: PLR0913
167
+ self,
168
+ key: Any,
169
+ value: Any,
170
+ expire: ExpireTime = ...,
171
+ read: bool = ...,
172
+ tag: Tag = ...,
173
+ retry: bool = ...,
174
+ ) -> bool: ...
175
+ def incr(
176
+ self, key: Any, delta: int = ..., default: int = ..., retry: bool = ...
177
+ ) -> int: ...
178
+ def decr(
179
+ self, key: Any, delta: int = ..., default: int = ..., retry: bool = ...
180
+ ) -> int: ...
181
+ def get( # noqa: PLR0913
182
+ self,
183
+ key: Any,
184
+ default: Any = ...,
185
+ read: bool = ...,
186
+ expire_time: bool = ...,
187
+ tag: bool = ...,
188
+ retry: bool = ...,
189
+ store_time: bool = ...,
190
+ ) -> Incomplete: ...
191
+ def __getitem__(self, key: Any) -> Any: ...
192
+ def __contains__(self, key: Any) -> bool: ...
193
+ def pop(
194
+ self,
195
+ key: Any,
196
+ default: Any = ...,
197
+ expire_time: bool = ...,
198
+ tag: bool = ...,
199
+ retry: bool = ...,
200
+ ) -> Incomplete: ...
201
+ def delete(self, key: Any, retry: bool = ...) -> bool: ...
202
+ def create_tag_index(self) -> None: ...
203
+ def drop_tag_index(self) -> None: ...
204
+ def check(
205
+ self, fix: bool = ..., retry: bool = ...
206
+ ) -> list[warnings.WarningMessage]: ...
207
+ def evict(self, tag: str, retry: bool = ...) -> int: ...
208
+ def cull(self, retry: bool = ...) -> int: ...
209
+ def clear(self, retry: bool = ...) -> int: ...
210
+ def __iter__(self) -> Iterator[Any]: ...
211
+ def __reversed__(self) -> Iterator[Any]: ...
212
+ def stats(
213
+ self, enable: bool = ..., reset: bool = ...
214
+ ) -> tuple[Any, Any]: ...
215
+ def volume(self) -> int: ...
216
+ def close(self) -> None: ...
217
+ def __enter__(self) -> Self: ...
218
+ def __exit__(self, *exception: object) -> None: ...
219
+ def __len__(self) -> int: ...
220
+ def memoize(
221
+ self,
222
+ name: str | None = ...,
223
+ typed: bool = ...,
224
+ expire: ExpireTime = ...,
225
+ tag: Tag = ...,
226
+ ignore: Ignore = ...,
227
+ ) -> Callable[[Callable[_P, _T]], Memoized[_P, _T]]: ...
@@ -0,0 +1,322 @@
1
+ """Disk that stores numpy arrays without pickling them."""
2
+
3
+ import io
4
+ import math
5
+ import pickle
6
+ import warnings
7
+
8
+ from .core import SAFE_PICKLE_NAMES, Disk, SafeUnpickler, _ReaderMissing
9
+
10
+ # Pickle protocol 5 is the first that stores a buffer in place, so that
11
+ # writing an array copies it once, into the row, and reading a file-backed
12
+ # entry needs no second copy of it.
13
+ _MIN_PROTOCOL = 5
14
+
15
+ # The numpy scalar types that are stored, by the name written to the row, with
16
+ # the Python type that ``item()`` gives for each and the reader insists on. The
17
+ # names do not vary by platform or numpy version, unlike the types' own names.
18
+ _SCALARS = {
19
+ 'bool': ('bool_', bool),
20
+ 'int8': ('int8', int),
21
+ 'int16': ('int16', int),
22
+ 'int32': ('int32', int),
23
+ 'int64': ('int64', int),
24
+ 'uint8': ('uint8', int),
25
+ 'uint16': ('uint16', int),
26
+ 'uint32': ('uint32', int),
27
+ 'uint64': ('uint64', int),
28
+ 'float16': ('float16', float),
29
+ 'float32': ('float32', float),
30
+ 'float64': ('float64', float),
31
+ 'complex64': ('complex64', complex),
32
+ 'complex128': ('complex128', complex),
33
+ 'str': ('str_', str),
34
+ 'bytes': ('bytes_', bytes),
35
+ }
36
+
37
+
38
+ def _import_numpy():
39
+ # pylint: disable=import-error,import-outside-toplevel
40
+ try:
41
+ import numpy
42
+ except ImportError as error:
43
+ raise _ReaderMissing(
44
+ 'SafeDisk needs numpy to read this entry'
45
+ ) from error
46
+ return numpy
47
+
48
+
49
+ def _read_header(numpy, header, size):
50
+ """Return shape, Fortran order and dtype from the ``.npy`` `header`.
51
+
52
+ Raise UnpicklingError unless the header is consistent with `size` bytes
53
+ of data. The sizes are checked because a forged shape must not decide
54
+ what gets read. Anything that goes wrong while parsing, including a
55
+ MemoryError from the Python parser on a crafted header, which
56
+ ``on_error='miss'`` would not treat as an unreadable entry, is reported
57
+ as UnpicklingError.
58
+
59
+ """
60
+ # pylint: disable=broad-except,no-member
61
+ try:
62
+ reader = io.BytesIO(header)
63
+ version = numpy.lib.format.read_magic(reader)
64
+ if version == (1, 0):
65
+ fields = numpy.lib.format.read_array_header_1_0(reader)
66
+ elif version == (2, 0):
67
+ fields = numpy.lib.format.read_array_header_2_0(reader)
68
+ else:
69
+ raise ValueError('unsupported .npy version %r' % (version,))
70
+ except Exception as error:
71
+ raise pickle.UnpicklingError(
72
+ 'SafeDisk refuses the array header'
73
+ ) from error
74
+
75
+ shape, fortran, dtype = fields
76
+ count = math.prod(shape)
77
+
78
+ # A zero-width dtype would let any shape match any amount of data.
79
+ if (
80
+ dtype.hasobject
81
+ or reader.tell() != len(header)
82
+ or dtype.itemsize * count != size
83
+ or (count and not dtype.itemsize)
84
+ ):
85
+ raise pickle.UnpicklingError('SafeDisk refuses the array header')
86
+
87
+ return shape, fortran, dtype
88
+
89
+
90
+ def _pack_array(numpy, array):
91
+ """Return the ``.npy`` header of `array` and a buffer of its data.
92
+
93
+ The buffer is the array's own memory, so pickle copies it once, into
94
+ the row, as a bytearray. An array that is not contiguous or not
95
+ writable is copied first, since pickle stores a read-only buffer as
96
+ bytes, which the reader refuses.
97
+
98
+ """
99
+ # pylint: disable=no-member
100
+ flags = array.flags
101
+ if not (flags.writeable and (flags.c_contiguous or flags.f_contiguous)):
102
+ array = array.copy(order='A')
103
+
104
+ buffer = io.BytesIO()
105
+ try:
106
+ with warnings.catch_warnings():
107
+ # numpy warns about dtypes it would save as pickles or in a
108
+ # format the reader refuses; both are refused below.
109
+ warnings.simplefilter('ignore')
110
+ # Version 1 holds every header the reader accepts: numpy refuses
111
+ # a header over 10000 bytes, and version 1 holds 65535.
112
+ fields = numpy.lib.format.header_data_from_array_1_0(array)
113
+ numpy.lib.format.write_array_header_1_0(buffer, fields)
114
+ header = buffer.getvalue()
115
+ _read_header(numpy, header, array.nbytes)
116
+ except (ValueError, pickle.UnpicklingError) as error:
117
+ raise TypeError(
118
+ 'SafeDisk cannot store an array of dtype %s' % (array.dtype,)
119
+ ) from error
120
+
121
+ return header, pickle.PickleBuffer(array)
122
+
123
+
124
+ def _unpack_array(header, data):
125
+ """Return the array in the bytearray `data`, which it keeps as its data.
126
+
127
+ A view takes no second copy of the array, and stays writable because
128
+ `data` is. `header` and `data` may have been tampered with.
129
+
130
+ """
131
+ # pylint: disable=broad-except
132
+ numpy = _import_numpy()
133
+ shape, fortran, dtype = _read_header(numpy, header, len(data))
134
+
135
+ try:
136
+ if not math.prod(shape):
137
+ return numpy.empty(shape, dtype)
138
+ array = numpy.frombuffer(data, dtype)
139
+ return array.reshape(shape, order='F' if fortran else 'C')
140
+ except Exception as error:
141
+ raise pickle.UnpicklingError(
142
+ 'SafeDisk refuses the array data'
143
+ ) from error
144
+
145
+
146
+ def _scalar_name(scalar):
147
+ """Return the name to store numpy `scalar` under, or raise TypeError."""
148
+ dtype = scalar.dtype
149
+ name = {'U': 'str', 'S': 'bytes'}.get(dtype.kind, dtype.name)
150
+ if name not in _SCALARS:
151
+ raise TypeError(
152
+ 'SafeDisk cannot store a scalar of type numpy.%s'
153
+ % (type(scalar).__name__,)
154
+ )
155
+ return name
156
+
157
+
158
+ def _unpack_scalar(name, value):
159
+ """Return the numpy scalar `value` of type `name`, which may be forged.
160
+
161
+ The value must be of the Python type the name is built from: numpy
162
+ would otherwise build something else, such as ``bytes_(n)``, which is
163
+ `n` zero bytes.
164
+
165
+ """
166
+ # pylint: disable=broad-except,unidiomatic-typecheck
167
+ entry = _SCALARS.get(name) if type(name) is str else None
168
+ if entry is None or type(value) is not entry[1]:
169
+ raise pickle.UnpicklingError('SafeDisk refuses a scalar')
170
+
171
+ numpy = _import_numpy()
172
+
173
+ try:
174
+ return getattr(numpy, entry[0])(value)
175
+ except Exception as error:
176
+ raise pickle.UnpicklingError('SafeDisk refuses a scalar') from error
177
+
178
+
179
+ class _ArrayPickler(pickle.Pickler):
180
+ """Pickler that writes arrays and numpy scalars as persistent IDs."""
181
+
182
+ def __init__(self, file, protocol, numpy):
183
+ super().__init__(file, protocol=protocol)
184
+ self._numpy = numpy
185
+ self._arrays = {}
186
+ self._scalar_names = {}
187
+ self.reduced = False
188
+
189
+ def reducer_override(self, obj):
190
+ """Note that an object was reduced, which can look up a name.
191
+
192
+ CPython skips this hook only for None, booleans and exact ints,
193
+ floats, bytes, strings, dicts, sets, frozensets, lists and tuples,
194
+ none of which pickle a name, so every pickle that looks one up is
195
+ noted here and scanned.
196
+
197
+ """
198
+ # pylint: disable=unused-argument
199
+ self.reduced = True
200
+ return NotImplemented
201
+
202
+ def persistent_id(self, obj):
203
+ # pylint: disable=unidiomatic-typecheck
204
+ numpy = self._numpy
205
+ if isinstance(obj, numpy.ndarray):
206
+ if type(obj) is not numpy.ndarray:
207
+ raise TypeError(
208
+ 'SafeDisk cannot store a subclass of ndarray: %s.%s'
209
+ % (type(obj).__module__, type(obj).__qualname__)
210
+ )
211
+ # pickle's own memo skips persistent IDs, and a value may hand it
212
+ # temporary arrays. The buffer in the ID keeps `obj` alive, so
213
+ # its id is not reused while this memo holds the ID.
214
+ pid = self._arrays.get(id(obj))
215
+ if pid is None:
216
+ pid = self._arrays[id(obj)] = ('npy',) + _pack_array(
217
+ numpy, obj
218
+ )
219
+ return pid
220
+ if isinstance(obj, numpy.generic):
221
+ kind = type(obj)
222
+ name = self._scalar_names.get(kind)
223
+ if name is None:
224
+ name = self._scalar_names[kind] = _scalar_name(obj)
225
+ value = obj.item()
226
+ if isinstance(value, numpy.generic):
227
+ raise TypeError(
228
+ 'SafeDisk cannot store a scalar of type numpy.%s'
229
+ % (kind.__name__,)
230
+ )
231
+ return ('scalar', name, value)
232
+ return None
233
+
234
+
235
+ class _ArrayUnpickler(SafeUnpickler):
236
+ """Unpickler that reads back what _ArrayPickler wrote.
237
+
238
+ Array state is rebuilt from the checked ``.npy`` header alone: the dtype
239
+ slot of a trailing ``BUILD`` accepts only a ``numpy.dtype``, which
240
+ neither the allowlist nor a persistent ID can produce.
241
+
242
+ """
243
+
244
+ def __init__(self, *args, **kwargs):
245
+ super().__init__(*args, **kwargs)
246
+ self._arrays = {}
247
+ self._strings = {}
248
+
249
+ def persistent_load(self, pid):
250
+ # pylint: disable=unidiomatic-typecheck
251
+ if type(pid) is tuple and len(pid) == 3:
252
+ if pid[0] == 'scalar':
253
+ value = pid[2]
254
+ if type(value) not in (str, bytes):
255
+ return _unpack_scalar(pid[1], value)
256
+ # A string the pickle memoized is copied into a numpy scalar
257
+ # once, however many IDs name it, as an array buffer is.
258
+ entry = self._strings.get(id(value))
259
+ if entry is None:
260
+ entry = (value, _unpack_scalar(pid[1], value))
261
+ self._strings[id(value)] = entry
262
+ return entry[1]
263
+ if pid[0] == 'npy':
264
+ header, payload = pid[1], pid[2]
265
+ if type(header) is bytes and type(payload) is bytearray:
266
+ # Decoding each buffer once keeps what a row can decode to
267
+ # within its own size, and shared arrays shared. Holding
268
+ # the pair keeps their ids from being reused.
269
+ key = (id(header), id(payload))
270
+ entry = self._arrays.get(key)
271
+ if entry is None:
272
+ array = _unpack_array(header, payload)
273
+ entry = (header, payload, array)
274
+ self._arrays[key] = entry
275
+ return entry[2]
276
+ raise pickle.UnpicklingError('SafeDisk refuses a persistent ID')
277
+
278
+
279
+ def unpickler(disk, file):
280
+ """Return the unpickler of `disk`, a SafeDisk with ``safe_numpy`` set."""
281
+ return _ArrayUnpickler(
282
+ file,
283
+ SAFE_PICKLE_NAMES,
284
+ disk.safe_types,
285
+ disk.safe_modules,
286
+ )
287
+
288
+
289
+ def store(disk, numpy, value, key):
290
+ """Return the Cache table fields of `value`, which may hold numpy data.
291
+
292
+ `disk` is a SafeDisk with ``safe_numpy`` set and `numpy` the imported
293
+ module. A value with no numpy in it is stored as an ordinary pickle.
294
+
295
+ """
296
+ # pylint: disable=protected-access
297
+ try:
298
+ disk._check(value, 'value')
299
+ except TypeError:
300
+ pass
301
+ else:
302
+ return Disk.store(disk, value, False, key=key)
303
+
304
+ buffer = io.BytesIO()
305
+ protocol = max(disk.pickle_protocol, _MIN_PROTOCOL)
306
+ pickler = _ArrayPickler(buffer, protocol, numpy)
307
+
308
+ try:
309
+ pickler.dump(value)
310
+ except (pickle.PicklingError, AttributeError) as error:
311
+ raise TypeError(
312
+ 'SafeDisk cannot store value: %s' % (error,)
313
+ ) from error
314
+
315
+ data = buffer.getvalue()
316
+
317
+ # Only a reduced object can look up a name, and scanning the pickle
318
+ # would copy every array in it.
319
+ if pickler.reduced:
320
+ disk._check_pickle(data, value, 'value')
321
+
322
+ return disk._store_pickle(data, value, key)
@@ -0,0 +1,8 @@
1
+ from typing import IO, Any
2
+
3
+ from diskcache_community.core import SafeDisk, SafeUnpickler
4
+
5
+ def unpickler(disk: SafeDisk, file: IO[bytes]) -> SafeUnpickler: ...
6
+ def store(
7
+ disk: SafeDisk, numpy: Any, value: Any, key: Any
8
+ ) -> tuple[int, int, str | None, Any]: ...