numpy-cache 0.1.0__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {numpy_cache-0.1.0/src/numpy_cache.egg-info → numpy_cache-0.1.2}/PKG-INFO +20 -18
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/README.md +18 -17
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/csrc/cache_module.c +163 -83
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/pyproject.toml +6 -2
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/setup.py +6 -1
- numpy_cache-0.1.2/src/numpy_cache/__init__.py +29 -0
- numpy_cache-0.1.2/src/numpy_cache/__main__.py +41 -0
- {numpy_cache-0.1.0 → numpy_cache-0.1.2/src/numpy_cache.egg-info}/PKG-INFO +20 -18
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/src/numpy_cache.egg-info/SOURCES.txt +1 -0
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/src/numpy_cache.egg-info/requires.txt +1 -0
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/tests/test_cache.py +193 -0
- numpy_cache-0.1.0/src/numpy_cache/__init__.py +0 -25
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/LICENSE +0 -0
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/setup.cfg +0 -0
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/src/numpy_cache.egg-info/dependency_links.txt +0 -0
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/src/numpy_cache.egg-info/not-zip-safe +0 -0
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/src/numpy_cache.egg-info/top_level.txt +0 -0
- {numpy_cache-0.1.0 → numpy_cache-0.1.2}/tests/test_benchmarks.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: numpy-cache
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: High-performance LZ4 cache for NumPy arrays
|
|
5
5
|
Author-email: "@macht1212" <nasimov.alexander@gmail.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -18,6 +18,7 @@ Requires-Python: <4.0.0,>=3.12
|
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: LICENSE
|
|
20
20
|
Requires-Dist: numpy<3.0.0,>=2.5.2
|
|
21
|
+
Requires-Dist: click<9.0.0,>=8.4.2
|
|
21
22
|
Dynamic: license-file
|
|
22
23
|
|
|
23
24
|
# NumPy Cache – Fast LZ4-Based Caching for NumPy Arrays
|
|
@@ -132,7 +133,8 @@ save(slice_arr, 'slice.npc') # Handles automatically
|
|
|
132
133
|
## 📊 Benchmarks
|
|
133
134
|
### Test system:
|
|
134
135
|
|
|
135
|
-
- Ubuntu 24.04.4 LTS, 12th Gen Intel i5-1235U (
|
|
136
|
+
- Ubuntu 24.04.4 LTS, 12th Gen Intel i5-1235U (10 cores), 16 GB RAM, SSD
|
|
137
|
+
- MacOS 15.7.2, Apple Silicon M1 (8 cores), 8 GB RAM, SSD
|
|
136
138
|
- Python 3.12, NumPy 2.5.2, LZ4 1.9.4
|
|
137
139
|
|
|
138
140
|
All arrays are `float32`. Sizes:
|
|
@@ -142,25 +144,25 @@ All arrays are `float32`. Sizes:
|
|
|
142
144
|
|
|
143
145
|
### Write Performance (time in μs)
|
|
144
146
|
|
|
145
|
-
| Method|
|
|
146
|
-
|
|
147
|
-
| `np.save`| 89.6| 879.6|
|
|
148
|
-
| `np.savez`| 117.4| 1 083.0|
|
|
149
|
-
| `np.savez_compressed`| 1 056.1| 28 921.5|
|
|
150
|
-
| **numpy_cache (accel=1)**| **71.8**| **759.4**|
|
|
151
|
-
| **numpy_cache (accel=4)**| **70.1**| **755.6**|
|
|
152
|
-
| **numpy_cache (accel=16)**| **74.9**| **738.9**|
|
|
147
|
+
| Method| Intel i5 <br> 0.04 MB| Intel i5 <br> 1 MB| Apple M1 <br> 0.04 MB| Apple M1 <br> 1 MB|
|
|
148
|
+
|-------|----------|--------|--------|------|
|
|
149
|
+
| `np.save`| 89.6| 879.6| 86.2 | 548.1 |
|
|
150
|
+
| `np.savez`| 117.4| 1 083.0| 94.5 | 554.4 |
|
|
151
|
+
| `np.savez_compressed`| 1 056.1| 28 921.5| 1 034.0 | 32 211.2 |
|
|
152
|
+
| **numpy_cache (accel=1)**| **71.8**| **759.4**| **67.8** | **770.1** |
|
|
153
|
+
| **numpy_cache (accel=4)**| **70.1**| **755.6**| **64.9** | **635.4** |
|
|
154
|
+
| **numpy_cache (accel=16)**| **74.9**| **738.9**| **68.0** | **655.2** |
|
|
153
155
|
|
|
154
156
|
### Read Performance (time in μs)
|
|
155
157
|
|
|
156
|
-
| Method|
|
|
157
|
-
|
|
158
|
-
| `np.save`| 41.7| 91.9|
|
|
159
|
-
| `np.savez`| 83.0| 401.1|
|
|
160
|
-
| `np.savez_compressed`| 267.5| 4 858.8|
|
|
161
|
-
| **numpy_cache (accel=1)**| **19.2**| **385.0**|
|
|
162
|
-
| **numpy_cache (accel=4)**| **17.5**| **421.0**|
|
|
163
|
-
| **numpy_cache (accel=16)**| **17.0**| **474.5**|
|
|
158
|
+
| Method| Intel i5 <br> 0.04 MB| Intel i5 <br> 1 MB| Apple M1 <br> 0.04 MB| Apple M1 <br> 1 MB|
|
|
159
|
+
|-------|----------|--------|--------|------|
|
|
160
|
+
| `np.save`| 41.7| 91.9| 44.8 | 77.3 |
|
|
161
|
+
| `np.savez`| 83.0| 401.1| 95.1 | 225.2 |
|
|
162
|
+
| `np.savez_compressed`| 267.5| 4 858.8| 193.7 | 2 876.7 |
|
|
163
|
+
| **numpy_cache (accel=1)**| **19.2**| **385.0**| **32.7** | **327.1** |
|
|
164
|
+
| **numpy_cache (accel=4)**| **17.5**| **421.0**| **23.8** | **665.8** |
|
|
165
|
+
| **numpy_cache (accel=16)**| **17.0**| **474.5**| **19.7** | **187.1** |
|
|
164
166
|
|
|
165
167
|
### File Size Comparison (1 MB array)
|
|
166
168
|
- `np.save` / `np.savez`: ~1.0 MB
|
|
@@ -110,7 +110,8 @@ save(slice_arr, 'slice.npc') # Handles automatically
|
|
|
110
110
|
## 📊 Benchmarks
|
|
111
111
|
### Test system:
|
|
112
112
|
|
|
113
|
-
- Ubuntu 24.04.4 LTS, 12th Gen Intel i5-1235U (
|
|
113
|
+
- Ubuntu 24.04.4 LTS, 12th Gen Intel i5-1235U (10 cores), 16 GB RAM, SSD
|
|
114
|
+
- MacOS 15.7.2, Apple Silicon M1 (8 cores), 8 GB RAM, SSD
|
|
114
115
|
- Python 3.12, NumPy 2.5.2, LZ4 1.9.4
|
|
115
116
|
|
|
116
117
|
All arrays are `float32`. Sizes:
|
|
@@ -120,25 +121,25 @@ All arrays are `float32`. Sizes:
|
|
|
120
121
|
|
|
121
122
|
### Write Performance (time in μs)
|
|
122
123
|
|
|
123
|
-
| Method|
|
|
124
|
-
|
|
125
|
-
| `np.save`| 89.6| 879.6|
|
|
126
|
-
| `np.savez`| 117.4| 1 083.0|
|
|
127
|
-
| `np.savez_compressed`| 1 056.1| 28 921.5|
|
|
128
|
-
| **numpy_cache (accel=1)**| **71.8**| **759.4**|
|
|
129
|
-
| **numpy_cache (accel=4)**| **70.1**| **755.6**|
|
|
130
|
-
| **numpy_cache (accel=16)**| **74.9**| **738.9**|
|
|
124
|
+
| Method| Intel i5 <br> 0.04 MB| Intel i5 <br> 1 MB| Apple M1 <br> 0.04 MB| Apple M1 <br> 1 MB|
|
|
125
|
+
|-------|----------|--------|--------|------|
|
|
126
|
+
| `np.save`| 89.6| 879.6| 86.2 | 548.1 |
|
|
127
|
+
| `np.savez`| 117.4| 1 083.0| 94.5 | 554.4 |
|
|
128
|
+
| `np.savez_compressed`| 1 056.1| 28 921.5| 1 034.0 | 32 211.2 |
|
|
129
|
+
| **numpy_cache (accel=1)**| **71.8**| **759.4**| **67.8** | **770.1** |
|
|
130
|
+
| **numpy_cache (accel=4)**| **70.1**| **755.6**| **64.9** | **635.4** |
|
|
131
|
+
| **numpy_cache (accel=16)**| **74.9**| **738.9**| **68.0** | **655.2** |
|
|
131
132
|
|
|
132
133
|
### Read Performance (time in μs)
|
|
133
134
|
|
|
134
|
-
| Method|
|
|
135
|
-
|
|
136
|
-
| `np.save`| 41.7| 91.9|
|
|
137
|
-
| `np.savez`| 83.0| 401.1|
|
|
138
|
-
| `np.savez_compressed`| 267.5| 4 858.8|
|
|
139
|
-
| **numpy_cache (accel=1)**| **19.2**| **385.0**|
|
|
140
|
-
| **numpy_cache (accel=4)**| **17.5**| **421.0**|
|
|
141
|
-
| **numpy_cache (accel=16)**| **17.0**| **474.5**|
|
|
135
|
+
| Method| Intel i5 <br> 0.04 MB| Intel i5 <br> 1 MB| Apple M1 <br> 0.04 MB| Apple M1 <br> 1 MB|
|
|
136
|
+
|-------|----------|--------|--------|------|
|
|
137
|
+
| `np.save`| 41.7| 91.9| 44.8 | 77.3 |
|
|
138
|
+
| `np.savez`| 83.0| 401.1| 95.1 | 225.2 |
|
|
139
|
+
| `np.savez_compressed`| 267.5| 4 858.8| 193.7 | 2 876.7 |
|
|
140
|
+
| **numpy_cache (accel=1)**| **19.2**| **385.0**| **32.7** | **327.1** |
|
|
141
|
+
| **numpy_cache (accel=4)**| **17.5**| **421.0**| **23.8** | **665.8** |
|
|
142
|
+
| **numpy_cache (accel=16)**| **17.0**| **474.5**| **19.7** | **187.1** |
|
|
142
143
|
|
|
143
144
|
### File Size Comparison (1 MB array)
|
|
144
145
|
- `np.save` / `np.savez`: ~1.0 MB
|
|
@@ -140,6 +140,84 @@ static int read_cache_file(const char* path, CacheHeader* header, void** compres
|
|
|
140
140
|
return 0;
|
|
141
141
|
}
|
|
142
142
|
|
|
143
|
+
static PyObject* shape_to_tuple(const CacheHeader* header) {
|
|
144
|
+
PyObject* shape_tuple = PyTuple_New(header->ndim);
|
|
145
|
+
if (!shape_tuple) return NULL;
|
|
146
|
+
|
|
147
|
+
for (uint32_t i = 0; i < header->ndim; ++i) {
|
|
148
|
+
PyObject* item = PyLong_FromUnsignedLongLong(header->shape[i]);
|
|
149
|
+
if (!item) {
|
|
150
|
+
Py_DECREF(shape_tuple);
|
|
151
|
+
return NULL;
|
|
152
|
+
}
|
|
153
|
+
PyTuple_SetItem(shape_tuple, i, item);
|
|
154
|
+
}
|
|
155
|
+
return shape_tuple;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
static PyArrayObject* create_array_from_header(const CacheHeader* header, void* data) {
|
|
159
|
+
PyArray_Descr* descr = PyArray_DescrFromType((int)header->dtype);
|
|
160
|
+
if (!descr) {
|
|
161
|
+
PyErr_SetString(PyExc_RuntimeError, "Unsupported dtype");
|
|
162
|
+
return NULL;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
npy_intp dims[MAX_DIMS];
|
|
166
|
+
for (int i = 0; i < (int)header->ndim; ++i) {
|
|
167
|
+
dims[i] = (npy_intp)header->shape[i];
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
PyArrayObject* arr = (PyArrayObject*)PyArray_NewFromDescr(
|
|
171
|
+
&PyArray_Type,
|
|
172
|
+
descr,
|
|
173
|
+
(int)header->ndim,
|
|
174
|
+
dims,
|
|
175
|
+
NULL,
|
|
176
|
+
data,
|
|
177
|
+
NPY_ARRAY_OWNDATA,
|
|
178
|
+
NULL
|
|
179
|
+
);
|
|
180
|
+
|
|
181
|
+
if (!arr) {
|
|
182
|
+
PyErr_SetString(PyExc_RuntimeError, "Failed to create NumPy array");
|
|
183
|
+
}
|
|
184
|
+
return arr;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
static int validate_header_size(const CacheHeader* header) {
|
|
188
|
+
if (header->uncompressed_size > MAX_SIZE) {
|
|
189
|
+
PyErr_SetString(PyExc_RuntimeError, "Uncompressed size too large (>1GB)");
|
|
190
|
+
return -1;
|
|
191
|
+
}
|
|
192
|
+
if (header->uncompressed_size > INT_MAX) {
|
|
193
|
+
PyErr_SetString(PyExc_RuntimeError, "Uncompressed size too large for LZ4");
|
|
194
|
+
return -1;
|
|
195
|
+
}
|
|
196
|
+
return 0;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
static PyObject* header_to_dict(const CacheHeader* header) {
|
|
200
|
+
PyObject* dict = PyDict_New();
|
|
201
|
+
if (!dict) return NULL;
|
|
202
|
+
|
|
203
|
+
PyDict_SetItemString(dict, "magic", PyLong_FromUnsignedLong(header->magic));
|
|
204
|
+
PyDict_SetItemString(dict, "version", PyLong_FromUnsignedLong(header->version));
|
|
205
|
+
PyDict_SetItemString(dict, "ndim", PyLong_FromUnsignedLong(header->ndim));
|
|
206
|
+
PyDict_SetItemString(dict, "dtype", PyLong_FromUnsignedLong(header->dtype));
|
|
207
|
+
PyDict_SetItemString(dict, "uncompressed_size", PyLong_FromUnsignedLongLong(header->uncompressed_size));
|
|
208
|
+
PyDict_SetItemString(dict, "compressed_size", PyLong_FromUnsignedLongLong(header->compressed_size));
|
|
209
|
+
|
|
210
|
+
PyObject* shape_tuple = shape_to_tuple(header);
|
|
211
|
+
if (!shape_tuple) {
|
|
212
|
+
Py_DECREF(dict);
|
|
213
|
+
return NULL;
|
|
214
|
+
}
|
|
215
|
+
PyDict_SetItemString(dict, "shape", shape_tuple);
|
|
216
|
+
Py_DECREF(shape_tuple);
|
|
217
|
+
|
|
218
|
+
return (PyObject*)dict;
|
|
219
|
+
}
|
|
220
|
+
|
|
143
221
|
static PyObject* cache_save(PyObject* self, PyObject* args, PyObject* kwargs) {
|
|
144
222
|
PyObject* obj = NULL;
|
|
145
223
|
const char* path = NULL;
|
|
@@ -236,93 +314,91 @@ static PyObject* cache_save(PyObject* self, PyObject* args, PyObject* kwargs) {
|
|
|
236
314
|
}
|
|
237
315
|
|
|
238
316
|
static PyObject* cache_load(PyObject* self, PyObject* args) {
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
free(compressed_data);
|
|
250
|
-
return NULL;
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
if (header.uncompressed_size > INT_MAX) {
|
|
254
|
-
PyErr_SetString(PyExc_RuntimeError, "Uncompressed size too large for LZ4");
|
|
255
|
-
free(compressed_data);
|
|
256
|
-
return NULL;
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
if (header.uncompressed_size == 0) {
|
|
260
|
-
free(compressed_data);
|
|
261
|
-
PyArray_Descr* descr = PyArray_DescrFromType((int)header.dtype);
|
|
262
|
-
if (!descr) {
|
|
263
|
-
PyErr_SetString(PyExc_RuntimeError, "Unsupported dtype");
|
|
317
|
+
const char* path = NULL;
|
|
318
|
+
if (!PyArg_ParseTuple(args, "s", &path)) return NULL;
|
|
319
|
+
|
|
320
|
+
CacheHeader header;
|
|
321
|
+
void* compressed_data = NULL;
|
|
322
|
+
|
|
323
|
+
if (read_cache_file(path, &header, &compressed_data) != 0) return NULL;
|
|
324
|
+
|
|
325
|
+
if (validate_header_size(&header) != 0) {
|
|
326
|
+
free(compressed_data);
|
|
264
327
|
return NULL;
|
|
265
328
|
}
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
329
|
+
|
|
330
|
+
if (header.uncompressed_size == 0) {
|
|
331
|
+
free(compressed_data);
|
|
332
|
+
PyArray_Descr* descr = PyArray_DescrFromType((int)header.dtype);
|
|
333
|
+
if (!descr) {
|
|
334
|
+
PyErr_SetString(PyExc_RuntimeError, "Unsupported dtype");
|
|
335
|
+
return NULL;
|
|
336
|
+
}
|
|
337
|
+
npy_intp dims[MAX_DIMS];
|
|
338
|
+
for (int i = 0; i < (int)header.ndim; ++i)
|
|
339
|
+
dims[i] = (npy_intp)header.shape[i];
|
|
340
|
+
return (PyObject*)PyArray_Zeros((int)header.ndim, dims, descr, 0);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
void* decompressed_data = malloc(header.uncompressed_size);
|
|
344
|
+
if (!decompressed_data) {
|
|
345
|
+
PyErr_NoMemory();
|
|
346
|
+
free(compressed_data);
|
|
347
|
+
return NULL;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
int decompressed_size = LZ4_decompress_safe(
|
|
351
|
+
(const char*)compressed_data,
|
|
352
|
+
(char*)decompressed_data,
|
|
353
|
+
(int)header.compressed_size,
|
|
354
|
+
(int)header.uncompressed_size
|
|
355
|
+
);
|
|
275
356
|
free(compressed_data);
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
PyArray_Descr* descr = PyArray_DescrFromType((int)header.dtype);
|
|
298
|
-
if (!descr) {
|
|
299
|
-
PyErr_SetString(PyExc_RuntimeError, "Unsupported dtype");
|
|
300
|
-
free(decompressed_data);
|
|
301
|
-
return NULL;
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
npy_intp dims[MAX_DIMS];
|
|
305
|
-
for (int i = 0; i < (int)header.ndim; ++i)
|
|
306
|
-
dims[i] = (npy_intp)header.shape[i];
|
|
307
|
-
|
|
308
|
-
PyArrayObject* arr = (PyArrayObject*)PyArray_NewFromDescr(
|
|
309
|
-
&PyArray_Type,
|
|
310
|
-
descr,
|
|
311
|
-
(int)header.ndim,
|
|
312
|
-
dims,
|
|
313
|
-
NULL,
|
|
314
|
-
decompressed_data,
|
|
315
|
-
NPY_ARRAY_OWNDATA,
|
|
316
|
-
NULL
|
|
317
|
-
);
|
|
318
|
-
|
|
319
|
-
if (!arr) {
|
|
320
|
-
PyErr_SetString(PyExc_RuntimeError, "Failed to create NumPy array");
|
|
321
|
-
free(decompressed_data);
|
|
322
|
-
return NULL;
|
|
323
|
-
}
|
|
357
|
+
|
|
358
|
+
if (decompressed_size < 0) {
|
|
359
|
+
PyErr_SetString(PyExc_RuntimeError, "LZ4 decompression failed");
|
|
360
|
+
free(decompressed_data);
|
|
361
|
+
return NULL;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
if ((uint64_t)decompressed_size != header.uncompressed_size) {
|
|
365
|
+
PyErr_SetString(PyExc_RuntimeError, "Decompressed size mismatch");
|
|
366
|
+
free(decompressed_data);
|
|
367
|
+
return NULL;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
PyArrayObject* arr = create_array_from_header(&header, decompressed_data);
|
|
371
|
+
if (!arr) {
|
|
372
|
+
free(decompressed_data);
|
|
373
|
+
return NULL;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
return (PyObject*)arr;
|
|
377
|
+
}
|
|
324
378
|
|
|
325
|
-
|
|
379
|
+
static PyObject* inspect_header(PyObject* self, PyObject* args) {
|
|
380
|
+
const char* path = NULL;
|
|
381
|
+
if (!PyArg_ParseTuple(args, "s", &path)) return NULL;
|
|
382
|
+
|
|
383
|
+
FILE* f = fopen(path, "rb");
|
|
384
|
+
if (!f) {
|
|
385
|
+
PyErr_Format(PyExc_IOError, "Cannot open file for reading: %s", strerror(errno));
|
|
386
|
+
return NULL;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
CacheHeader header;
|
|
390
|
+
if (fread(&header, sizeof(CacheHeader), 1, f) != 1) {
|
|
391
|
+
PyErr_SetString(PyExc_IOError, "Failed to read header (file too short or corrupt)");
|
|
392
|
+
fclose(f);
|
|
393
|
+
return NULL;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
if (fclose(f) != 0) {
|
|
397
|
+
PyErr_Format(PyExc_OSError, "Failed to close file: %s", strerror(errno));
|
|
398
|
+
return NULL;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
return header_to_dict(&header);
|
|
326
402
|
}
|
|
327
403
|
|
|
328
404
|
static PyMethodDef CacheMethods[] = {
|
|
@@ -348,6 +424,10 @@ static PyMethodDef CacheMethods[] = {
|
|
|
348
424
|
"Raises:\n"
|
|
349
425
|
" RuntimeError on invalid or corrupted file."
|
|
350
426
|
},
|
|
427
|
+
{"inspect", inspect_header, METH_VARARGS,
|
|
428
|
+
"Read and display the header of a cache file without loading the data.\n"
|
|
429
|
+
"Returns:\n A dictionary with magic, version, ndim, dtype, shape, sizes.\n"
|
|
430
|
+
},
|
|
351
431
|
{NULL, NULL, 0, NULL}
|
|
352
432
|
};
|
|
353
433
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "numpy-cache"
|
|
3
|
-
version = "0.1.
|
|
3
|
+
version = "0.1.2"
|
|
4
4
|
description = "High-performance LZ4 cache for NumPy arrays"
|
|
5
5
|
authors = [
|
|
6
6
|
{name = "@macht1212", email = "nasimov.alexander@gmail.com"}
|
|
@@ -20,6 +20,7 @@ classifiers = [
|
|
|
20
20
|
]
|
|
21
21
|
dependencies = [
|
|
22
22
|
"numpy>=2.5.2,<3.0.0",
|
|
23
|
+
"click (>=8.4.2,<9.0.0)",
|
|
23
24
|
]
|
|
24
25
|
|
|
25
26
|
[project.urls]
|
|
@@ -40,4 +41,7 @@ build-backend = "setuptools.build_meta"
|
|
|
40
41
|
|
|
41
42
|
[tool.setuptools.packages.find]
|
|
42
43
|
where = ["src"]
|
|
43
|
-
include = ["numpy_cache*"]
|
|
44
|
+
include = ["numpy_cache*"]
|
|
45
|
+
|
|
46
|
+
[tool.poetry.scripts]
|
|
47
|
+
numpy-cache = "numpy_cache.__main__:cli"
|
|
@@ -81,9 +81,14 @@ print("===========================")
|
|
|
81
81
|
|
|
82
82
|
setup(
|
|
83
83
|
name="numpy-cache",
|
|
84
|
-
version="0.1.
|
|
84
|
+
version="0.1.2",
|
|
85
85
|
packages=["numpy_cache"],
|
|
86
86
|
package_dir={"": "src"},
|
|
87
87
|
ext_modules=[cache_module],
|
|
88
88
|
zip_safe=False,
|
|
89
|
+
entry_points={
|
|
90
|
+
'console_scripts': [
|
|
91
|
+
'numpy-cache=numpy_cache.__main__:cli',
|
|
92
|
+
],
|
|
93
|
+
},
|
|
89
94
|
)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from .__main__ import cli
|
|
2
|
+
from ._cache import (
|
|
3
|
+
ACCELERATION_DEFAULT,
|
|
4
|
+
ACCELERATION_MAX,
|
|
5
|
+
ACCELERATION_MIN,
|
|
6
|
+
MAGIC,
|
|
7
|
+
MAX_DIMS,
|
|
8
|
+
MAX_SIZE,
|
|
9
|
+
VERSION,
|
|
10
|
+
inspect,
|
|
11
|
+
load,
|
|
12
|
+
save,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
__version__ = "0.1.0"
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"ACCELERATION_DEFAULT",
|
|
19
|
+
"ACCELERATION_MAX",
|
|
20
|
+
"ACCELERATION_MIN",
|
|
21
|
+
"MAGIC",
|
|
22
|
+
"MAX_DIMS",
|
|
23
|
+
"MAX_SIZE",
|
|
24
|
+
"VERSION",
|
|
25
|
+
"cli",
|
|
26
|
+
"inspect",
|
|
27
|
+
"load",
|
|
28
|
+
"save",
|
|
29
|
+
]
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
|
|
4
|
+
import click
|
|
5
|
+
|
|
6
|
+
from ._cache import inspect
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class Header:
|
|
11
|
+
magic: str
|
|
12
|
+
version: int
|
|
13
|
+
ndim: int
|
|
14
|
+
dtype: int
|
|
15
|
+
uncompressed_size: int
|
|
16
|
+
compressed_size: int
|
|
17
|
+
shape: tuple[int, ...]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@click.group()
|
|
21
|
+
def cli(): ...
|
|
22
|
+
|
|
23
|
+
@cli.command(name="inspect")
|
|
24
|
+
@click.argument("path")
|
|
25
|
+
def show_file_headers(path: str):
|
|
26
|
+
if not os.path.exists(path):
|
|
27
|
+
raise FileNotFoundError(f"File not find on {path=}.")
|
|
28
|
+
|
|
29
|
+
info = Header(**inspect(path))
|
|
30
|
+
|
|
31
|
+
click.echo(
|
|
32
|
+
f"Magic: {info.magic}\nVersion: {info.version}\n"
|
|
33
|
+
f"NDIM: {info.ndim}\ndType: {info.dtype}\n"
|
|
34
|
+
f"Uncompressed size: {info.uncompressed_size}\n"
|
|
35
|
+
f"Compressed size: {info.compressed_size}\n"
|
|
36
|
+
f"Shape: {info.shape}"
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
if __name__ == "__main__":
|
|
41
|
+
cli()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: numpy-cache
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: High-performance LZ4 cache for NumPy arrays
|
|
5
5
|
Author-email: "@macht1212" <nasimov.alexander@gmail.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -18,6 +18,7 @@ Requires-Python: <4.0.0,>=3.12
|
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: LICENSE
|
|
20
20
|
Requires-Dist: numpy<3.0.0,>=2.5.2
|
|
21
|
+
Requires-Dist: click<9.0.0,>=8.4.2
|
|
21
22
|
Dynamic: license-file
|
|
22
23
|
|
|
23
24
|
# NumPy Cache – Fast LZ4-Based Caching for NumPy Arrays
|
|
@@ -132,7 +133,8 @@ save(slice_arr, 'slice.npc') # Handles automatically
|
|
|
132
133
|
## 📊 Benchmarks
|
|
133
134
|
### Test system:
|
|
134
135
|
|
|
135
|
-
- Ubuntu 24.04.4 LTS, 12th Gen Intel i5-1235U (
|
|
136
|
+
- Ubuntu 24.04.4 LTS, 12th Gen Intel i5-1235U (10 cores), 16 GB RAM, SSD
|
|
137
|
+
- MacOS 15.7.2, Apple Silicon M1 (8 cores), 8 GB RAM, SSD
|
|
136
138
|
- Python 3.12, NumPy 2.5.2, LZ4 1.9.4
|
|
137
139
|
|
|
138
140
|
All arrays are `float32`. Sizes:
|
|
@@ -142,25 +144,25 @@ All arrays are `float32`. Sizes:
|
|
|
142
144
|
|
|
143
145
|
### Write Performance (time in μs)
|
|
144
146
|
|
|
145
|
-
| Method|
|
|
146
|
-
|
|
147
|
-
| `np.save`| 89.6| 879.6|
|
|
148
|
-
| `np.savez`| 117.4| 1 083.0|
|
|
149
|
-
| `np.savez_compressed`| 1 056.1| 28 921.5|
|
|
150
|
-
| **numpy_cache (accel=1)**| **71.8**| **759.4**|
|
|
151
|
-
| **numpy_cache (accel=4)**| **70.1**| **755.6**|
|
|
152
|
-
| **numpy_cache (accel=16)**| **74.9**| **738.9**|
|
|
147
|
+
| Method| Intel i5 <br> 0.04 MB| Intel i5 <br> 1 MB| Apple M1 <br> 0.04 MB| Apple M1 <br> 1 MB|
|
|
148
|
+
|-------|----------|--------|--------|------|
|
|
149
|
+
| `np.save`| 89.6| 879.6| 86.2 | 548.1 |
|
|
150
|
+
| `np.savez`| 117.4| 1 083.0| 94.5 | 554.4 |
|
|
151
|
+
| `np.savez_compressed`| 1 056.1| 28 921.5| 1 034.0 | 32 211.2 |
|
|
152
|
+
| **numpy_cache (accel=1)**| **71.8**| **759.4**| **67.8** | **770.1** |
|
|
153
|
+
| **numpy_cache (accel=4)**| **70.1**| **755.6**| **64.9** | **635.4** |
|
|
154
|
+
| **numpy_cache (accel=16)**| **74.9**| **738.9**| **68.0** | **655.2** |
|
|
153
155
|
|
|
154
156
|
### Read Performance (time in μs)
|
|
155
157
|
|
|
156
|
-
| Method|
|
|
157
|
-
|
|
158
|
-
| `np.save`| 41.7| 91.9|
|
|
159
|
-
| `np.savez`| 83.0| 401.1|
|
|
160
|
-
| `np.savez_compressed`| 267.5| 4 858.8|
|
|
161
|
-
| **numpy_cache (accel=1)**| **19.2**| **385.0**|
|
|
162
|
-
| **numpy_cache (accel=4)**| **17.5**| **421.0**|
|
|
163
|
-
| **numpy_cache (accel=16)**| **17.0**| **474.5**|
|
|
158
|
+
| Method| Intel i5 <br> 0.04 MB| Intel i5 <br> 1 MB| Apple M1 <br> 0.04 MB| Apple M1 <br> 1 MB|
|
|
159
|
+
|-------|----------|--------|--------|------|
|
|
160
|
+
| `np.save`| 41.7| 91.9| 44.8 | 77.3 |
|
|
161
|
+
| `np.savez`| 83.0| 401.1| 95.1 | 225.2 |
|
|
162
|
+
| `np.savez_compressed`| 267.5| 4 858.8| 193.7 | 2 876.7 |
|
|
163
|
+
| **numpy_cache (accel=1)**| **19.2**| **385.0**| **32.7** | **327.1** |
|
|
164
|
+
| **numpy_cache (accel=4)**| **17.5**| **421.0**| **23.8** | **665.8** |
|
|
165
|
+
| **numpy_cache (accel=16)**| **17.0**| **474.5**| **19.7** | **187.1** |
|
|
164
166
|
|
|
165
167
|
### File Size Comparison (1 MB array)
|
|
166
168
|
- `np.save` / `np.savez`: ~1.0 MB
|
|
@@ -11,6 +11,7 @@ from numpy_cache._cache import (
|
|
|
11
11
|
MAX_DIMS,
|
|
12
12
|
MAX_SIZE,
|
|
13
13
|
VERSION,
|
|
14
|
+
inspect,
|
|
14
15
|
load,
|
|
15
16
|
save,
|
|
16
17
|
)
|
|
@@ -217,3 +218,195 @@ def test_metadata_preserved(temp_file):
|
|
|
217
218
|
assert loaded.shape == (2, 3, 4)
|
|
218
219
|
assert loaded.dtype == np.int16
|
|
219
220
|
np.testing.assert_array_equal(arr, loaded)
|
|
221
|
+
|
|
222
|
+
def test_inspect_valid_file():
|
|
223
|
+
arr = np.random.randn(100, 100).astype(np.float32)
|
|
224
|
+
with tempfile.NamedTemporaryFile(suffix=".npc", delete=False) as f:
|
|
225
|
+
path = f.name
|
|
226
|
+
|
|
227
|
+
try:
|
|
228
|
+
save(arr, path)
|
|
229
|
+
info = inspect(path)
|
|
230
|
+
|
|
231
|
+
assert isinstance(info, dict)
|
|
232
|
+
assert "magic" in info
|
|
233
|
+
assert "version" in info
|
|
234
|
+
assert "ndim" in info
|
|
235
|
+
assert "dtype" in info
|
|
236
|
+
assert "uncompressed_size" in info
|
|
237
|
+
assert "compressed_size" in info
|
|
238
|
+
assert "shape" in info
|
|
239
|
+
|
|
240
|
+
assert info["magic"] == 0x4C5A4E43 # "LZNC"
|
|
241
|
+
assert info["version"] == 1
|
|
242
|
+
assert info["ndim"] == 2
|
|
243
|
+
assert info["dtype"] == np.dtype(np.float32).num # 11
|
|
244
|
+
assert info["uncompressed_size"] >= 0
|
|
245
|
+
assert info["shape"] == (100, 100)
|
|
246
|
+
assert info["compressed_size"] > 0
|
|
247
|
+
|
|
248
|
+
finally:
|
|
249
|
+
if os.path.exists(path):
|
|
250
|
+
os.remove(path)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def test_inspect_empty_array():
|
|
254
|
+
arr = np.array([]).astype(np.float32)
|
|
255
|
+
with tempfile.NamedTemporaryFile(suffix=".npc", delete=False) as f:
|
|
256
|
+
path = f.name
|
|
257
|
+
|
|
258
|
+
try:
|
|
259
|
+
save(arr, path)
|
|
260
|
+
info = inspect(path)
|
|
261
|
+
|
|
262
|
+
assert info["ndim"] == 1
|
|
263
|
+
assert info["shape"] == (0,)
|
|
264
|
+
assert info["uncompressed_size"] == 0
|
|
265
|
+
assert info["compressed_size"] == 0
|
|
266
|
+
|
|
267
|
+
finally:
|
|
268
|
+
if os.path.exists(path):
|
|
269
|
+
os.remove(path)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def test_inspect_different_dtypes():
|
|
273
|
+
dtypes = [
|
|
274
|
+
np.float32, np.float64,
|
|
275
|
+
np.int8, np.int16, np.int32, np.int64,
|
|
276
|
+
np.uint8, np.uint16, np.uint32, np.uint64,
|
|
277
|
+
np.bool_
|
|
278
|
+
]
|
|
279
|
+
|
|
280
|
+
for dtype in dtypes:
|
|
281
|
+
if dtype == np.bool_:
|
|
282
|
+
arr = np.random.choice([True, False], size=(10, 10))
|
|
283
|
+
else:
|
|
284
|
+
arr = np.random.randint(0, 100, size=(10, 10)).astype(dtype)
|
|
285
|
+
|
|
286
|
+
with tempfile.NamedTemporaryFile(suffix=".npc", delete=False) as f:
|
|
287
|
+
path = f.name
|
|
288
|
+
|
|
289
|
+
try:
|
|
290
|
+
save(arr, path)
|
|
291
|
+
info = inspect(path)
|
|
292
|
+
assert info["dtype"] == np.dtype(dtype).num
|
|
293
|
+
assert info["shape"] == (10, 10)
|
|
294
|
+
assert info["uncompressed_size"] > 0
|
|
295
|
+
finally:
|
|
296
|
+
if os.path.exists(path):
|
|
297
|
+
os.remove(path)
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def test_inspect_multidimensional():
|
|
301
|
+
shapes = [(10, 10, 10), (5, 6, 7, 8), (3, 4, 5, 6, 7)]
|
|
302
|
+
|
|
303
|
+
for shape in shapes:
|
|
304
|
+
arr = np.random.randn(*shape).astype(np.float32)
|
|
305
|
+
with tempfile.NamedTemporaryFile(suffix=".npc", delete=False) as f:
|
|
306
|
+
path = f.name
|
|
307
|
+
|
|
308
|
+
try:
|
|
309
|
+
save(arr, path)
|
|
310
|
+
info = inspect(path)
|
|
311
|
+
assert info["ndim"] == len(shape)
|
|
312
|
+
assert info["shape"] == shape
|
|
313
|
+
assert info["uncompressed_size"] == np.prod(shape) * 4
|
|
314
|
+
finally:
|
|
315
|
+
if os.path.exists(path):
|
|
316
|
+
os.remove(path)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def test_inspect_non_contiguous():
|
|
320
|
+
arr = np.random.randn(50, 50).astype(np.float32)
|
|
321
|
+
sliced = arr[::2, ::2]
|
|
322
|
+
|
|
323
|
+
with tempfile.NamedTemporaryFile(suffix=".npc", delete=False) as f:
|
|
324
|
+
path = f.name
|
|
325
|
+
|
|
326
|
+
try:
|
|
327
|
+
save(sliced, path)
|
|
328
|
+
info = inspect(path)
|
|
329
|
+
assert info["shape"] == (25, 25)
|
|
330
|
+
assert info["uncompressed_size"] == 25 * 25 * 4
|
|
331
|
+
finally:
|
|
332
|
+
if os.path.exists(path):
|
|
333
|
+
os.remove(path)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def test_inspect_nonexistent_file():
|
|
337
|
+
with pytest.raises(IOError, match="Cannot open file for reading"):
|
|
338
|
+
inspect("/nonexistent/file/path.npc")
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def test_inspect_invalid_file():
|
|
342
|
+
with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f:
|
|
343
|
+
path = f.name
|
|
344
|
+
f.write(b"This is not a numpy-cache file")
|
|
345
|
+
|
|
346
|
+
try:
|
|
347
|
+
with pytest.raises(OSError, match="Failed to read header"):
|
|
348
|
+
inspect(path)
|
|
349
|
+
finally:
|
|
350
|
+
if os.path.exists(path):
|
|
351
|
+
os.remove(path)
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def test_inspect_corrupted_header():
|
|
355
|
+
arr = np.random.randn(10, 10).astype(np.float32)
|
|
356
|
+
with tempfile.NamedTemporaryFile(suffix=".npc", delete=False) as f:
|
|
357
|
+
path = f.name
|
|
358
|
+
|
|
359
|
+
try:
|
|
360
|
+
save(arr, path)
|
|
361
|
+
|
|
362
|
+
with open(path, "r+b") as f:
|
|
363
|
+
f.seek(0)
|
|
364
|
+
f.write(b"\x00\x01\x02\x03")
|
|
365
|
+
|
|
366
|
+
info = inspect(path)
|
|
367
|
+
assert isinstance(info, dict)
|
|
368
|
+
finally:
|
|
369
|
+
if os.path.exists(path):
|
|
370
|
+
os.remove(path)
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def test_inspect_very_large_ndim():
|
|
374
|
+
arr = np.random.randn(10, 10).astype(np.float32)
|
|
375
|
+
with tempfile.NamedTemporaryFile(suffix=".npc", delete=False) as f:
|
|
376
|
+
path = f.name
|
|
377
|
+
|
|
378
|
+
try:
|
|
379
|
+
save(arr, path)
|
|
380
|
+
|
|
381
|
+
with open(path, "r+b") as f:
|
|
382
|
+
f.seek(88)
|
|
383
|
+
f.write(b"\xFF\xFF\xFF\xFF")
|
|
384
|
+
|
|
385
|
+
try:
|
|
386
|
+
info = inspect(path)
|
|
387
|
+
assert info["ndim"] > 1000
|
|
388
|
+
except (MemoryError, OverflowError, ValueError, SystemError):
|
|
389
|
+
pass
|
|
390
|
+
|
|
391
|
+
finally:
|
|
392
|
+
if os.path.exists(path):
|
|
393
|
+
os.remove(path)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def test_inspect_acceleration_parameter():
|
|
397
|
+
accelerations = [1, 4, 16]
|
|
398
|
+
|
|
399
|
+
for accel in accelerations:
|
|
400
|
+
arr = np.random.randn(100, 100).astype(np.float32)
|
|
401
|
+
with tempfile.NamedTemporaryFile(suffix=".npc", delete=False) as f:
|
|
402
|
+
path = f.name
|
|
403
|
+
|
|
404
|
+
try:
|
|
405
|
+
save(arr, path, acceleration=accel)
|
|
406
|
+
info = inspect(path)
|
|
407
|
+
assert info["shape"] == (100, 100)
|
|
408
|
+
assert info["dtype"] == np.dtype(np.float32).num
|
|
409
|
+
assert info["compressed_size"] > 0
|
|
410
|
+
finally:
|
|
411
|
+
if os.path.exists(path):
|
|
412
|
+
os.remove(path)
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
from ._cache import (
|
|
2
|
-
ACCELERATION_DEFAULT,
|
|
3
|
-
ACCELERATION_MAX,
|
|
4
|
-
ACCELERATION_MIN,
|
|
5
|
-
MAGIC,
|
|
6
|
-
MAX_DIMS,
|
|
7
|
-
MAX_SIZE,
|
|
8
|
-
VERSION,
|
|
9
|
-
load,
|
|
10
|
-
save,
|
|
11
|
-
)
|
|
12
|
-
|
|
13
|
-
__version__ = "0.1.0"
|
|
14
|
-
|
|
15
|
-
__all__ = [
|
|
16
|
-
"ACCELERATION_DEFAULT",
|
|
17
|
-
"ACCELERATION_MAX",
|
|
18
|
-
"ACCELERATION_MIN",
|
|
19
|
-
"MAGIC",
|
|
20
|
-
"MAX_DIMS",
|
|
21
|
-
"MAX_SIZE",
|
|
22
|
-
"VERSION",
|
|
23
|
-
"load",
|
|
24
|
-
"save"
|
|
25
|
-
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|