fastconstmap 0.10.0__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/PKG-INFO +24 -2
- fastconstmap-0.10.0/fastconstmap.egg-info/PKG-INFO → fastconstmap-0.11.0/README.md +22 -19
- fastconstmap-0.10.0/README.md → fastconstmap-0.11.0/fastconstmap.egg-info/PKG-INFO +41 -1
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/fastconstmap.egg-info/SOURCES.txt +2 -1
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/pyproject.toml +4 -1
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/src/fastconstmap_module.c +122 -18
- fastconstmap-0.11.0/tests/test_threading.py +149 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/LICENSE +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/MANIFEST.in +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/benchmarks/bench_batch.c +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/benchmarks/benchmark.py +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/examples/example.py +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/examples/shared_memory_example.py +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/fastconstmap/__init__.py +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/fastconstmap/_fastconstmap.pyi +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/fastconstmap/py.typed +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/fastconstmap.egg-info/dependency_links.txt +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/fastconstmap.egg-info/top_level.txt +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/setup.cfg +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/setup.py +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/src/constmap.c +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/src/constmap.h +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/src/third_party/xxhash/LICENSE +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/src/third_party/xxhash/xxhash.h +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/tests/test_constmap.py +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/tests/test_interop.py +0 -0
- {fastconstmap-0.10.0 → fastconstmap-0.11.0}/tests/test_shared_memory.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fastconstmap
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Fast, immutable, compact map from strings to uint64 (binary fuse filter).
|
|
5
5
|
Author-email: Daniel Lemire <daniel@lemire.me>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -8,6 +8,7 @@ Project-URL: Homepage, https://github.com/lemire/fastconstmap
|
|
|
8
8
|
Project-URL: Source, https://github.com/lemire/fastconstmap
|
|
9
9
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
10
|
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable
|
|
11
12
|
Classifier: Programming Language :: C
|
|
12
13
|
Classifier: Operating System :: OS Independent
|
|
13
14
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
@@ -297,7 +298,7 @@ Notes and constraints:
|
|
|
297
298
|
use. Drop the map (`del cm`) before calling `shm.close()`.
|
|
298
299
|
- The map is **immutable** — the intended pattern is *write once in the
|
|
299
300
|
producer, then only read in every process*. Concurrent readers need no
|
|
300
|
-
locking.
|
|
301
|
+
locking (see [Threads](#threads-and-free-threaded-python)).
|
|
301
302
|
- `from_buffer` verifies the magic bytes and the FNV-1a checksum, so a
|
|
302
303
|
truncated or corrupt block raises `ValueError` rather than returning
|
|
303
304
|
garbage.
|
|
@@ -313,6 +314,27 @@ Notes and constraints:
|
|
|
313
314
|
load on a big-endian host): it copies the data and the resulting map owns
|
|
314
315
|
its memory independently of the source buffer.
|
|
315
316
|
|
|
317
|
+
## Threads and free-threaded Python
|
|
318
|
+
|
|
319
|
+
The maps are immutable, so any number of threads can look up keys in the
|
|
320
|
+
same map at once, without locking. The extension supports free-threaded
|
|
321
|
+
Python (3.13t, 3.14t): it does not re-enable the GIL on import, and lookups
|
|
322
|
+
from several threads then run in parallel.
|
|
323
|
+
|
|
324
|
+
```python
|
|
325
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
326
|
+
|
|
327
|
+
m = ConstMap(d)
|
|
328
|
+
with ThreadPoolExecutor(8) as pool:
|
|
329
|
+
results = list(pool.map(m.get_many, batches))
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
Building a map from a dict that another thread is modifying is safe (on a
|
|
333
|
+
free-threaded build the map is built from a snapshot of the dict), and so is
|
|
334
|
+
a batched lookup over a list that another thread is modifying. `__init__` can only be called once on a
|
|
335
|
+
map; calling it again raises `TypeError` rather than replacing the data
|
|
336
|
+
under other readers.
|
|
337
|
+
|
|
316
338
|
## Interoperability with the Go and Rust implementations
|
|
317
339
|
|
|
318
340
|
A map saved by any of [`constmap`][constmap] (Go), [`rsconstmap`][rsconstmap]
|
|
@@ -1,21 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: fastconstmap
|
|
3
|
-
Version: 0.10.0
|
|
4
|
-
Summary: Fast, immutable, compact map from strings to uint64 (binary fuse filter).
|
|
5
|
-
Author-email: Daniel Lemire <daniel@lemire.me>
|
|
6
|
-
License: Apache-2.0
|
|
7
|
-
Project-URL: Homepage, https://github.com/lemire/fastconstmap
|
|
8
|
-
Project-URL: Source, https://github.com/lemire/fastconstmap
|
|
9
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
|
-
Classifier: Programming Language :: Python :: 3
|
|
11
|
-
Classifier: Programming Language :: C
|
|
12
|
-
Classifier: Operating System :: OS Independent
|
|
13
|
-
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
14
|
-
Requires-Python: >=3.8
|
|
15
|
-
Description-Content-Type: text/markdown
|
|
16
|
-
License-File: LICENSE
|
|
17
|
-
Dynamic: license-file
|
|
18
|
-
|
|
19
1
|
# fastconstmap
|
|
20
2
|
|
|
21
3
|
[](LICENSE)
|
|
@@ -297,7 +279,7 @@ Notes and constraints:
|
|
|
297
279
|
use. Drop the map (`del cm`) before calling `shm.close()`.
|
|
298
280
|
- The map is **immutable** — the intended pattern is *write once in the
|
|
299
281
|
producer, then only read in every process*. Concurrent readers need no
|
|
300
|
-
locking.
|
|
282
|
+
locking (see [Threads](#threads-and-free-threaded-python)).
|
|
301
283
|
- `from_buffer` verifies the magic bytes and the FNV-1a checksum, so a
|
|
302
284
|
truncated or corrupt block raises `ValueError` rather than returning
|
|
303
285
|
garbage.
|
|
@@ -313,6 +295,27 @@ Notes and constraints:
|
|
|
313
295
|
load on a big-endian host): it copies the data and the resulting map owns
|
|
314
296
|
its memory independently of the source buffer.
|
|
315
297
|
|
|
298
|
+
## Threads and free-threaded Python
|
|
299
|
+
|
|
300
|
+
The maps are immutable, so any number of threads can look up keys in the
|
|
301
|
+
same map at once, without locking. The extension supports free-threaded
|
|
302
|
+
Python (3.13t, 3.14t): it does not re-enable the GIL on import, and lookups
|
|
303
|
+
from several threads then run in parallel.
|
|
304
|
+
|
|
305
|
+
```python
|
|
306
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
307
|
+
|
|
308
|
+
m = ConstMap(d)
|
|
309
|
+
with ThreadPoolExecutor(8) as pool:
|
|
310
|
+
results = list(pool.map(m.get_many, batches))
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
Building a map from a dict that another thread is modifying is safe (on a
|
|
314
|
+
free-threaded build the map is built from a snapshot of the dict), and so is
|
|
315
|
+
a batched lookup over a list that another thread is modifying. `__init__` can only be called once on a
|
|
316
|
+
map; calling it again raises `TypeError` rather than replacing the data
|
|
317
|
+
under other readers.
|
|
318
|
+
|
|
316
319
|
## Interoperability with the Go and Rust implementations
|
|
317
320
|
|
|
318
321
|
A map saved by any of [`constmap`][constmap] (Go), [`rsconstmap`][rsconstmap]
|
|
@@ -1,3 +1,22 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fastconstmap
|
|
3
|
+
Version: 0.11.0
|
|
4
|
+
Summary: Fast, immutable, compact map from strings to uint64 (binary fuse filter).
|
|
5
|
+
Author-email: Daniel Lemire <daniel@lemire.me>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/lemire/fastconstmap
|
|
8
|
+
Project-URL: Source, https://github.com/lemire/fastconstmap
|
|
9
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable
|
|
12
|
+
Classifier: Programming Language :: C
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
15
|
+
Requires-Python: >=3.8
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
1
20
|
# fastconstmap
|
|
2
21
|
|
|
3
22
|
[](LICENSE)
|
|
@@ -279,7 +298,7 @@ Notes and constraints:
|
|
|
279
298
|
use. Drop the map (`del cm`) before calling `shm.close()`.
|
|
280
299
|
- The map is **immutable** — the intended pattern is *write once in the
|
|
281
300
|
producer, then only read in every process*. Concurrent readers need no
|
|
282
|
-
locking.
|
|
301
|
+
locking (see [Threads](#threads-and-free-threaded-python)).
|
|
283
302
|
- `from_buffer` verifies the magic bytes and the FNV-1a checksum, so a
|
|
284
303
|
truncated or corrupt block raises `ValueError` rather than returning
|
|
285
304
|
garbage.
|
|
@@ -295,6 +314,27 @@ Notes and constraints:
|
|
|
295
314
|
load on a big-endian host): it copies the data and the resulting map owns
|
|
296
315
|
its memory independently of the source buffer.
|
|
297
316
|
|
|
317
|
+
## Threads and free-threaded Python
|
|
318
|
+
|
|
319
|
+
The maps are immutable, so any number of threads can look up keys in the
|
|
320
|
+
same map at once, without locking. The extension supports free-threaded
|
|
321
|
+
Python (3.13t, 3.14t): it does not re-enable the GIL on import, and lookups
|
|
322
|
+
from several threads then run in parallel.
|
|
323
|
+
|
|
324
|
+
```python
|
|
325
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
326
|
+
|
|
327
|
+
m = ConstMap(d)
|
|
328
|
+
with ThreadPoolExecutor(8) as pool:
|
|
329
|
+
results = list(pool.map(m.get_many, batches))
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
Building a map from a dict that another thread is modifying is safe (on a
|
|
333
|
+
free-threaded build the map is built from a snapshot of the dict), and so is
|
|
334
|
+
a batched lookup over a list that another thread is modifying. `__init__` can only be called once on a
|
|
335
|
+
map; calling it again raises `TypeError` rather than replacing the data
|
|
336
|
+
under other readers.
|
|
337
|
+
|
|
298
338
|
## Interoperability with the Go and Rust implementations
|
|
299
339
|
|
|
300
340
|
A map saved by any of [`constmap`][constmap] (Go), [`rsconstmap`][rsconstmap]
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "fastconstmap"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.11.0"
|
|
8
8
|
description = "Fast, immutable, compact map from strings to uint64 (binary fuse filter)."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "Apache-2.0" }
|
|
@@ -13,6 +13,7 @@ requires-python = ">=3.8"
|
|
|
13
13
|
classifiers = [
|
|
14
14
|
"License :: OSI Approved :: Apache Software License",
|
|
15
15
|
"Programming Language :: Python :: 3",
|
|
16
|
+
"Programming Language :: Python :: Free Threading :: 3 - Stable",
|
|
16
17
|
"Programming Language :: C",
|
|
17
18
|
"Operating System :: OS Independent",
|
|
18
19
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
@@ -33,6 +34,8 @@ fastconstmap = ["py.typed"]
|
|
|
33
34
|
# our pytest suite — catches packaging regressions before they ship.
|
|
34
35
|
test-requires = "pytest"
|
|
35
36
|
test-command = "pytest {project}/tests"
|
|
37
|
+
# Also build the free-threaded 3.13t wheels (3.14t is built by default).
|
|
38
|
+
enable = ["cpython-freethreading"]
|
|
36
39
|
|
|
37
40
|
[tool.cibuildwheel.macos]
|
|
38
41
|
archs = ["x86_64", "universal2", "arm64"]
|
|
@@ -13,6 +13,35 @@
|
|
|
13
13
|
#include <stdint.h>
|
|
14
14
|
#include <string.h>
|
|
15
15
|
|
|
16
|
+
/* ------------------------------------------------------------------------- */
|
|
17
|
+
/* Free threading */
|
|
18
|
+
/* ------------------------------------------------------------------------- */
|
|
19
|
+
|
|
20
|
+
/* The maps are immutable once built, so lookups need no locking. What needs
|
|
21
|
+
* care on a free-threaded build is (1) the dict and lists we read, which
|
|
22
|
+
* another thread may be mutating, and (2) __init__, which installs the map.
|
|
23
|
+
* Critical sections only exist on 3.13+; with the GIL they are not needed. */
|
|
24
|
+
#ifndef Py_BEGIN_CRITICAL_SECTION
|
|
25
|
+
#define Py_BEGIN_CRITICAL_SECTION(op) {
|
|
26
|
+
#define Py_END_CRITICAL_SECTION() }
|
|
27
|
+
#endif
|
|
28
|
+
|
|
29
|
+
/* Like PySequence_Fast, but on a free-threaded build a list (which another
|
|
30
|
+
* thread could mutate while we read its items) is snapshotted into a tuple.
|
|
31
|
+
* Any other iterable already becomes a fresh, private list. */
|
|
32
|
+
static PyObject *fcm_sequence_fast(PyObject *obj, const char *msg) {
|
|
33
|
+
#ifdef Py_GIL_DISABLED
|
|
34
|
+
if (PyList_Check(obj)) return PyList_AsTuple(obj);
|
|
35
|
+
#endif
|
|
36
|
+
return PySequence_Fast(obj, msg);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
static int fcm_already_initialized(const char *type_name) {
|
|
40
|
+
PyErr_Format(PyExc_TypeError,
|
|
41
|
+
"%s is immutable: __init__ cannot be called again", type_name);
|
|
42
|
+
return -1;
|
|
43
|
+
}
|
|
44
|
+
|
|
16
45
|
/* ------------------------------------------------------------------------- */
|
|
17
46
|
/* Batched-lookup helpers */
|
|
18
47
|
/* ------------------------------------------------------------------------- */
|
|
@@ -134,12 +163,17 @@ static int fcm_dict_to_arrays(PyObject *dict,
|
|
|
134
163
|
PyErr_SetString(PyExc_TypeError, "expected a dict");
|
|
135
164
|
return -1;
|
|
136
165
|
}
|
|
137
|
-
Py_ssize_t n = PyDict_Size(dict);
|
|
138
|
-
if (n < 0) return -1;
|
|
139
|
-
|
|
140
166
|
fcm_key_t *keys = NULL;
|
|
141
167
|
uint64_t *vals = NULL;
|
|
142
168
|
PyObject *keep = NULL;
|
|
169
|
+
#ifdef Py_GIL_DISABLED
|
|
170
|
+
/* Iterate over a private copy: another thread may mutate `dict`. */
|
|
171
|
+
PyObject *copy = PyDict_Copy(dict);
|
|
172
|
+
if (!copy) return -1;
|
|
173
|
+
dict = copy;
|
|
174
|
+
#endif
|
|
175
|
+
Py_ssize_t n = PyDict_Size(dict);
|
|
176
|
+
if (n < 0) goto fail;
|
|
143
177
|
|
|
144
178
|
keep = PyTuple_New(n);
|
|
145
179
|
if (!keep) goto fail;
|
|
@@ -205,12 +239,18 @@ static int fcm_dict_to_arrays(PyObject *dict,
|
|
|
205
239
|
*out_keys = keys;
|
|
206
240
|
*out_values = vals;
|
|
207
241
|
*out_keep = keep;
|
|
242
|
+
#ifdef Py_GIL_DISABLED
|
|
243
|
+
Py_DECREF(copy);
|
|
244
|
+
#endif
|
|
208
245
|
return 0;
|
|
209
246
|
|
|
210
247
|
fail:
|
|
211
248
|
PyMem_Free(keys);
|
|
212
249
|
PyMem_Free(vals);
|
|
213
250
|
Py_XDECREF(keep);
|
|
251
|
+
#ifdef Py_GIL_DISABLED
|
|
252
|
+
Py_DECREF(copy);
|
|
253
|
+
#endif
|
|
214
254
|
return -1;
|
|
215
255
|
}
|
|
216
256
|
|
|
@@ -224,6 +264,8 @@ typedef struct {
|
|
|
224
264
|
/* When `view.obj` is non-NULL the map is a zero-copy view: `cm.data`
|
|
225
265
|
* points into `view`'s buffer and must NOT be freed. */
|
|
226
266
|
Py_buffer view;
|
|
267
|
+
/* Set once the map holds its data; it never changes afterwards. */
|
|
268
|
+
int initialized;
|
|
227
269
|
} PyConstMap;
|
|
228
270
|
|
|
229
271
|
/* Release whatever storage the map currently holds (owned heap data or a
|
|
@@ -251,11 +293,13 @@ static int PyConstMap_init(PyConstMap *self, PyObject *args, PyObject *kwargs) {
|
|
|
251
293
|
PyObject *keep = NULL;
|
|
252
294
|
if (fcm_dict_to_arrays(dict, &n, &keys, &vals, &keep) < 0) return -1;
|
|
253
295
|
|
|
254
|
-
|
|
255
|
-
|
|
296
|
+
/* Build into a local map and install it only if the object holds none:
|
|
297
|
+
* replacing a map would free memory other threads may be reading. */
|
|
298
|
+
fcm_constmap_t built;
|
|
299
|
+
memset(&built, 0, sizeof(built));
|
|
256
300
|
int rc;
|
|
257
301
|
Py_BEGIN_ALLOW_THREADS
|
|
258
|
-
rc = fcm_constmap_new_with_hash(&
|
|
302
|
+
rc = fcm_constmap_new_with_hash(&built, keys, vals, n, hash);
|
|
259
303
|
Py_END_ALLOW_THREADS
|
|
260
304
|
|
|
261
305
|
PyMem_Free(keys);
|
|
@@ -263,6 +307,18 @@ static int PyConstMap_init(PyConstMap *self, PyObject *args, PyObject *kwargs) {
|
|
|
263
307
|
Py_DECREF(keep);
|
|
264
308
|
|
|
265
309
|
if (rc != FCM_OK) { fcm_set_python_error(rc); return -1; }
|
|
310
|
+
int installed = 0;
|
|
311
|
+
Py_BEGIN_CRITICAL_SECTION(self);
|
|
312
|
+
if (!self->initialized) {
|
|
313
|
+
self->cm = built;
|
|
314
|
+
self->initialized = 1;
|
|
315
|
+
installed = 1;
|
|
316
|
+
}
|
|
317
|
+
Py_END_CRITICAL_SECTION();
|
|
318
|
+
if (!installed) {
|
|
319
|
+
fcm_constmap_free(&built);
|
|
320
|
+
return fcm_already_initialized("ConstMap");
|
|
321
|
+
}
|
|
266
322
|
return 0;
|
|
267
323
|
}
|
|
268
324
|
|
|
@@ -295,7 +351,7 @@ static PyObject *PyConstMap_get(PyConstMap *self, PyObject *arg) {
|
|
|
295
351
|
* gain over a loop of single lookups comes from once the map no longer fits in
|
|
296
352
|
* cache. */
|
|
297
353
|
static PyObject *PyConstMap_get_many(PyConstMap *self, PyObject *arg) {
|
|
298
|
-
PyObject *seq =
|
|
354
|
+
PyObject *seq = fcm_sequence_fast(arg, "get_many() requires an iterable of keys");
|
|
299
355
|
if (!seq) return NULL;
|
|
300
356
|
Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
|
|
301
357
|
PyObject **items = PySequence_Fast_ITEMS(seq);
|
|
@@ -332,7 +388,7 @@ static PyObject *PyConstMap_get_many_into(PyConstMap *self, PyObject *args) {
|
|
|
332
388
|
PyObject *keys_arg, *out_arg;
|
|
333
389
|
if (!PyArg_ParseTuple(args, "OO:get_many_into", &keys_arg, &out_arg)) return NULL;
|
|
334
390
|
|
|
335
|
-
PyObject *seq =
|
|
391
|
+
PyObject *seq = fcm_sequence_fast(keys_arg, "get_many_into() requires an iterable of keys");
|
|
336
392
|
if (!seq) return NULL;
|
|
337
393
|
Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
|
|
338
394
|
PyObject **items = PySequence_Fast_ITEMS(seq);
|
|
@@ -397,6 +453,7 @@ static PyObject *PyConstMap_deserialize(PyTypeObject *type, PyObject *arg) {
|
|
|
397
453
|
fcm_set_python_error(rc);
|
|
398
454
|
return NULL;
|
|
399
455
|
}
|
|
456
|
+
self->initialized = 1;
|
|
400
457
|
return (PyObject *)self;
|
|
401
458
|
}
|
|
402
459
|
|
|
@@ -446,6 +503,7 @@ static PyObject *PyConstMap_load(PyTypeObject *type, PyObject *arg) {
|
|
|
446
503
|
int rc = fcm_constmap_read(&self->cm, buf, (size_t)len);
|
|
447
504
|
PyMem_Free(buf);
|
|
448
505
|
if (rc != FCM_OK) { Py_DECREF(self); fcm_set_python_error(rc); return NULL; }
|
|
506
|
+
self->initialized = 1;
|
|
449
507
|
return (PyObject *)self;
|
|
450
508
|
}
|
|
451
509
|
|
|
@@ -497,6 +555,7 @@ static PyObject *PyConstMap_from_buffer(PyTypeObject *type, PyObject *arg) {
|
|
|
497
555
|
return NULL;
|
|
498
556
|
}
|
|
499
557
|
self->view = view; /* keep the buffer alive for the map's lifetime */
|
|
558
|
+
self->initialized = 1;
|
|
500
559
|
return (PyObject *)self;
|
|
501
560
|
}
|
|
502
561
|
|
|
@@ -571,6 +630,8 @@ typedef struct {
|
|
|
571
630
|
fcm_verified_constmap_t vm;
|
|
572
631
|
/* Non-NULL view.obj => zero-copy view: data/checks point into the buffer. */
|
|
573
632
|
Py_buffer view;
|
|
633
|
+
/* Set once the map holds its data; it never changes afterwards. */
|
|
634
|
+
int initialized;
|
|
574
635
|
} PyVerifiedConstMap;
|
|
575
636
|
|
|
576
637
|
static void pyverified_release(PyVerifiedConstMap *self) {
|
|
@@ -596,11 +657,13 @@ static int PyVerifiedConstMap_init(PyVerifiedConstMap *self, PyObject *args, PyO
|
|
|
596
657
|
PyObject *keep = NULL;
|
|
597
658
|
if (fcm_dict_to_arrays(dict, &n, &keys, &vals, &keep) < 0) return -1;
|
|
598
659
|
|
|
599
|
-
|
|
600
|
-
|
|
660
|
+
/* Build into a local map and install it only if the object holds none:
|
|
661
|
+
* replacing a map would free memory other threads may be reading. */
|
|
662
|
+
fcm_verified_constmap_t built;
|
|
663
|
+
memset(&built, 0, sizeof(built));
|
|
601
664
|
int rc;
|
|
602
665
|
Py_BEGIN_ALLOW_THREADS
|
|
603
|
-
rc = fcm_verified_constmap_new_with_hash(&
|
|
666
|
+
rc = fcm_verified_constmap_new_with_hash(&built, keys, vals, n, hash);
|
|
604
667
|
Py_END_ALLOW_THREADS
|
|
605
668
|
|
|
606
669
|
PyMem_Free(keys);
|
|
@@ -608,6 +671,18 @@ static int PyVerifiedConstMap_init(PyVerifiedConstMap *self, PyObject *args, PyO
|
|
|
608
671
|
Py_DECREF(keep);
|
|
609
672
|
|
|
610
673
|
if (rc != FCM_OK) { fcm_set_python_error(rc); return -1; }
|
|
674
|
+
int installed = 0;
|
|
675
|
+
Py_BEGIN_CRITICAL_SECTION(self);
|
|
676
|
+
if (!self->initialized) {
|
|
677
|
+
self->vm = built;
|
|
678
|
+
self->initialized = 1;
|
|
679
|
+
installed = 1;
|
|
680
|
+
}
|
|
681
|
+
Py_END_CRITICAL_SECTION();
|
|
682
|
+
if (!installed) {
|
|
683
|
+
fcm_verified_constmap_free(&built);
|
|
684
|
+
return fcm_already_initialized("VerifiedConstMap");
|
|
685
|
+
}
|
|
611
686
|
return 0;
|
|
612
687
|
}
|
|
613
688
|
|
|
@@ -680,7 +755,7 @@ static PyObject *PyVerifiedConstMap_get_many(PyVerifiedConstMap *self, PyObject
|
|
|
680
755
|
static char *kwlist[] = {"keys", "default", NULL};
|
|
681
756
|
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", kwlist, &arg, &default_)) return NULL;
|
|
682
757
|
|
|
683
|
-
PyObject *seq =
|
|
758
|
+
PyObject *seq = fcm_sequence_fast(arg, "get_many() requires an iterable of keys");
|
|
684
759
|
if (!seq) return NULL;
|
|
685
760
|
Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
|
|
686
761
|
PyObject **items = PySequence_Fast_ITEMS(seq);
|
|
@@ -723,7 +798,7 @@ static PyObject *PyVerifiedConstMap_get_many_into(PyVerifiedConstMap *self, PyOb
|
|
|
723
798
|
PyObject *keys_arg, *out_arg;
|
|
724
799
|
if (!PyArg_ParseTuple(args, "OO:get_many_into", &keys_arg, &out_arg)) return NULL;
|
|
725
800
|
|
|
726
|
-
PyObject *seq =
|
|
801
|
+
PyObject *seq = fcm_sequence_fast(keys_arg, "get_many_into() requires an iterable of keys");
|
|
727
802
|
if (!seq) return NULL;
|
|
728
803
|
Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
|
|
729
804
|
PyObject **items = PySequence_Fast_ITEMS(seq);
|
|
@@ -778,6 +853,7 @@ static PyObject *PyVerifiedConstMap_deserialize(PyTypeObject *type, PyObject *ar
|
|
|
778
853
|
int rc = fcm_verified_constmap_read(&self->vm, view.buf, (size_t)view.len);
|
|
779
854
|
PyBuffer_Release(&view);
|
|
780
855
|
if (rc != FCM_OK) { Py_DECREF(self); fcm_set_python_error(rc); return NULL; }
|
|
856
|
+
self->initialized = 1;
|
|
781
857
|
return (PyObject *)self;
|
|
782
858
|
}
|
|
783
859
|
|
|
@@ -824,6 +900,7 @@ static PyObject *PyVerifiedConstMap_load(PyTypeObject *type, PyObject *arg) {
|
|
|
824
900
|
int rc = fcm_verified_constmap_read(&self->vm, buf, (size_t)len);
|
|
825
901
|
PyMem_Free(buf);
|
|
826
902
|
if (rc != FCM_OK) { Py_DECREF(self); fcm_set_python_error(rc); return NULL; }
|
|
903
|
+
self->initialized = 1;
|
|
827
904
|
return (PyObject *)self;
|
|
828
905
|
}
|
|
829
906
|
|
|
@@ -870,6 +947,7 @@ static PyObject *PyVerifiedConstMap_from_buffer(PyTypeObject *type, PyObject *ar
|
|
|
870
947
|
return NULL;
|
|
871
948
|
}
|
|
872
949
|
self->view = view;
|
|
950
|
+
self->initialized = 1;
|
|
873
951
|
return (PyObject *)self;
|
|
874
952
|
}
|
|
875
953
|
|
|
@@ -948,6 +1026,8 @@ typedef struct {
|
|
|
948
1026
|
fcm_paired_verified_constmap_t vm;
|
|
949
1027
|
/* Non-NULL view.obj => zero-copy view: slots point into the buffer. */
|
|
950
1028
|
Py_buffer view;
|
|
1029
|
+
/* Set once the map holds its data; it never changes afterwards. */
|
|
1030
|
+
int initialized;
|
|
951
1031
|
} PyPairedVerifiedConstMap;
|
|
952
1032
|
|
|
953
1033
|
static void pypaired_release(PyPairedVerifiedConstMap *self) {
|
|
@@ -972,11 +1052,13 @@ static int PyPairedVerifiedConstMap_init(PyPairedVerifiedConstMap *self, PyObjec
|
|
|
972
1052
|
PyObject *keep = NULL;
|
|
973
1053
|
if (fcm_dict_to_arrays(dict, &n, &keys, &vals, &keep) < 0) return -1;
|
|
974
1054
|
|
|
975
|
-
|
|
976
|
-
|
|
1055
|
+
/* Build into a local map and install it only if the object holds none:
|
|
1056
|
+
* replacing a map would free memory other threads may be reading. */
|
|
1057
|
+
fcm_paired_verified_constmap_t built;
|
|
1058
|
+
memset(&built, 0, sizeof(built));
|
|
977
1059
|
int rc;
|
|
978
1060
|
Py_BEGIN_ALLOW_THREADS
|
|
979
|
-
rc = fcm_paired_verified_constmap_new_with_hash(&
|
|
1061
|
+
rc = fcm_paired_verified_constmap_new_with_hash(&built, keys, vals, n, hash);
|
|
980
1062
|
Py_END_ALLOW_THREADS
|
|
981
1063
|
|
|
982
1064
|
PyMem_Free(keys);
|
|
@@ -984,6 +1066,18 @@ static int PyPairedVerifiedConstMap_init(PyPairedVerifiedConstMap *self, PyObjec
|
|
|
984
1066
|
Py_DECREF(keep);
|
|
985
1067
|
|
|
986
1068
|
if (rc != FCM_OK) { fcm_set_python_error(rc); return -1; }
|
|
1069
|
+
int installed = 0;
|
|
1070
|
+
Py_BEGIN_CRITICAL_SECTION(self);
|
|
1071
|
+
if (!self->initialized) {
|
|
1072
|
+
self->vm = built;
|
|
1073
|
+
self->initialized = 1;
|
|
1074
|
+
installed = 1;
|
|
1075
|
+
}
|
|
1076
|
+
Py_END_CRITICAL_SECTION();
|
|
1077
|
+
if (!installed) {
|
|
1078
|
+
fcm_paired_verified_constmap_free(&built);
|
|
1079
|
+
return fcm_already_initialized("PairedVerifiedConstMap");
|
|
1080
|
+
}
|
|
987
1081
|
return 0;
|
|
988
1082
|
}
|
|
989
1083
|
|
|
@@ -1056,7 +1150,7 @@ static PyObject *PyPairedVerifiedConstMap_get_many(PyPairedVerifiedConstMap *sel
|
|
|
1056
1150
|
static char *kwlist[] = {"keys", "default", NULL};
|
|
1057
1151
|
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", kwlist, &arg, &default_)) return NULL;
|
|
1058
1152
|
|
|
1059
|
-
PyObject *seq =
|
|
1153
|
+
PyObject *seq = fcm_sequence_fast(arg, "get_many() requires an iterable of keys");
|
|
1060
1154
|
if (!seq) return NULL;
|
|
1061
1155
|
Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
|
|
1062
1156
|
PyObject **items = PySequence_Fast_ITEMS(seq);
|
|
@@ -1099,7 +1193,7 @@ static PyObject *PyPairedVerifiedConstMap_get_many_into(PyPairedVerifiedConstMap
|
|
|
1099
1193
|
PyObject *keys_arg, *out_arg;
|
|
1100
1194
|
if (!PyArg_ParseTuple(args, "OO:get_many_into", &keys_arg, &out_arg)) return NULL;
|
|
1101
1195
|
|
|
1102
|
-
PyObject *seq =
|
|
1196
|
+
PyObject *seq = fcm_sequence_fast(keys_arg, "get_many_into() requires an iterable of keys");
|
|
1103
1197
|
if (!seq) return NULL;
|
|
1104
1198
|
Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
|
|
1105
1199
|
PyObject **items = PySequence_Fast_ITEMS(seq);
|
|
@@ -1154,6 +1248,7 @@ static PyObject *PyPairedVerifiedConstMap_deserialize(PyTypeObject *type, PyObje
|
|
|
1154
1248
|
int rc = fcm_paired_verified_constmap_read(&self->vm, view.buf, (size_t)view.len);
|
|
1155
1249
|
PyBuffer_Release(&view);
|
|
1156
1250
|
if (rc != FCM_OK) { Py_DECREF(self); fcm_set_python_error(rc); return NULL; }
|
|
1251
|
+
self->initialized = 1;
|
|
1157
1252
|
return (PyObject *)self;
|
|
1158
1253
|
}
|
|
1159
1254
|
|
|
@@ -1200,6 +1295,7 @@ static PyObject *PyPairedVerifiedConstMap_load(PyTypeObject *type, PyObject *arg
|
|
|
1200
1295
|
int rc = fcm_paired_verified_constmap_read(&self->vm, buf, (size_t)len);
|
|
1201
1296
|
PyMem_Free(buf);
|
|
1202
1297
|
if (rc != FCM_OK) { Py_DECREF(self); fcm_set_python_error(rc); return NULL; }
|
|
1298
|
+
self->initialized = 1;
|
|
1203
1299
|
return (PyObject *)self;
|
|
1204
1300
|
}
|
|
1205
1301
|
|
|
@@ -1246,6 +1342,7 @@ static PyObject *PyPairedVerifiedConstMap_from_buffer(PyTypeObject *type, PyObje
|
|
|
1246
1342
|
return NULL;
|
|
1247
1343
|
}
|
|
1248
1344
|
self->view = view;
|
|
1345
|
+
self->initialized = 1;
|
|
1249
1346
|
return (PyObject *)self;
|
|
1250
1347
|
}
|
|
1251
1348
|
|
|
@@ -1334,6 +1431,13 @@ PyMODINIT_FUNC PyInit__fastconstmap(void) {
|
|
|
1334
1431
|
|
|
1335
1432
|
PyObject *m = PyModule_Create(&fastconstmap_module);
|
|
1336
1433
|
if (!m) return NULL;
|
|
1434
|
+
#ifdef Py_GIL_DISABLED
|
|
1435
|
+
/* Safe to run without the GIL (see "Free threading" above). */
|
|
1436
|
+
if (PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED) < 0) {
|
|
1437
|
+
Py_DECREF(m);
|
|
1438
|
+
return NULL;
|
|
1439
|
+
}
|
|
1440
|
+
#endif
|
|
1337
1441
|
|
|
1338
1442
|
Py_INCREF(&PyConstMapType);
|
|
1339
1443
|
if (PyModule_AddObject(m, "ConstMap", (PyObject *)&PyConstMapType) < 0) {
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Concurrent use of the maps. On a free-threaded build (python3.14t) these
|
|
2
|
+
run truly in parallel; with the GIL they still exercise the same code."""
|
|
3
|
+
import array
|
|
4
|
+
import sys
|
|
5
|
+
import sysconfig
|
|
6
|
+
import threading
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
from fastconstmap import NOT_FOUND, ConstMap, PairedVerifiedConstMap, VerifiedConstMap
|
|
11
|
+
|
|
12
|
+
N_THREADS = 8
|
|
13
|
+
TYPES = [ConstMap, VerifiedConstMap, PairedVerifiedConstMap]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def run_threads(target, n=N_THREADS):
|
|
17
|
+
barrier = threading.Barrier(n)
|
|
18
|
+
errors = []
|
|
19
|
+
|
|
20
|
+
def wrapper(i):
|
|
21
|
+
barrier.wait()
|
|
22
|
+
try:
|
|
23
|
+
target(i)
|
|
24
|
+
except BaseException as e: # pragma: no cover - reported below
|
|
25
|
+
errors.append(e)
|
|
26
|
+
|
|
27
|
+
threads = [threading.Thread(target=wrapper, args=(i,)) for i in range(n)]
|
|
28
|
+
for t in threads:
|
|
29
|
+
t.start()
|
|
30
|
+
for t in threads:
|
|
31
|
+
t.join()
|
|
32
|
+
if errors:
|
|
33
|
+
raise errors[0]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def make_data(n=20_000):
|
|
37
|
+
return {f"key-{i}": i * 7 for i in range(n)}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@pytest.mark.skipif(not sysconfig.get_config_var("Py_GIL_DISABLED"),
|
|
41
|
+
reason="needs a free-threaded build")
|
|
42
|
+
def test_gil_stays_disabled():
|
|
43
|
+
# Importing the extension must not re-enable the GIL.
|
|
44
|
+
assert not sys._is_gil_enabled()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@pytest.mark.parametrize("cls", TYPES)
|
|
48
|
+
def test_concurrent_lookups(cls):
|
|
49
|
+
data = make_data()
|
|
50
|
+
m = cls(data)
|
|
51
|
+
keys = list(data)
|
|
52
|
+
expected = [data[k] for k in keys]
|
|
53
|
+
|
|
54
|
+
def work(i):
|
|
55
|
+
for _ in range(3):
|
|
56
|
+
assert m.get_many(keys) == expected
|
|
57
|
+
out = array.array("Q", bytes(8 * len(keys)))
|
|
58
|
+
assert m.get_many_into(keys, out) == len(keys)
|
|
59
|
+
assert list(out) == expected
|
|
60
|
+
for k in keys[i::N_THREADS]:
|
|
61
|
+
assert m[k] == data[k]
|
|
62
|
+
|
|
63
|
+
run_threads(work)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@pytest.mark.parametrize("cls", [VerifiedConstMap, PairedVerifiedConstMap])
|
|
67
|
+
def test_concurrent_absent_keys(cls):
|
|
68
|
+
m = cls(make_data())
|
|
69
|
+
absent = [f"absent-{i}" for i in range(5_000)]
|
|
70
|
+
|
|
71
|
+
def work(i):
|
|
72
|
+
assert m.get_many(absent) == [None] * len(absent)
|
|
73
|
+
out = array.array("Q", bytes(8 * len(absent)))
|
|
74
|
+
m.get_many_into(absent, out)
|
|
75
|
+
assert set(out) == {NOT_FOUND}
|
|
76
|
+
|
|
77
|
+
run_threads(work)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@pytest.mark.parametrize("cls", TYPES)
|
|
81
|
+
def test_concurrent_construction(cls):
|
|
82
|
+
data = make_data(5_000)
|
|
83
|
+
blob = cls(data).to_bytes()
|
|
84
|
+
|
|
85
|
+
def work(i):
|
|
86
|
+
local = {f"{i}-{k}": v for k, v in data.items()}
|
|
87
|
+
m = cls(local)
|
|
88
|
+
assert m.get_many(list(local)) == list(local.values())
|
|
89
|
+
assert cls.from_bytes(blob).to_bytes() == blob
|
|
90
|
+
|
|
91
|
+
run_threads(work)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@pytest.mark.parametrize("cls", TYPES)
|
|
95
|
+
def test_lookup_while_list_mutates(cls):
|
|
96
|
+
# A key list shared with a thread that mutates it must not crash the
|
|
97
|
+
# lookup; the results are whatever keys the list held at the time.
|
|
98
|
+
data = make_data(2_000)
|
|
99
|
+
m = cls(data)
|
|
100
|
+
keys = list(data)
|
|
101
|
+
stop = threading.Event()
|
|
102
|
+
|
|
103
|
+
def mutate():
|
|
104
|
+
while not stop.is_set():
|
|
105
|
+
keys.append("x")
|
|
106
|
+
keys.pop()
|
|
107
|
+
keys.reverse()
|
|
108
|
+
|
|
109
|
+
t = threading.Thread(target=mutate)
|
|
110
|
+
t.start()
|
|
111
|
+
try:
|
|
112
|
+
for _ in range(200):
|
|
113
|
+
assert len(m.get_many(keys)) in (len(data), len(data) + 1)
|
|
114
|
+
finally:
|
|
115
|
+
stop.set()
|
|
116
|
+
t.join()
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@pytest.mark.parametrize("cls", TYPES)
|
|
120
|
+
def test_construct_while_dict_mutates(cls):
|
|
121
|
+
data = make_data(2_000)
|
|
122
|
+
stop = threading.Event()
|
|
123
|
+
|
|
124
|
+
def mutate():
|
|
125
|
+
while not stop.is_set():
|
|
126
|
+
data["extra"] = 1
|
|
127
|
+
del data["extra"]
|
|
128
|
+
|
|
129
|
+
t = threading.Thread(target=mutate)
|
|
130
|
+
t.start()
|
|
131
|
+
try:
|
|
132
|
+
for _ in range(50):
|
|
133
|
+
m = cls(data)
|
|
134
|
+
assert m["key-0"] == 0
|
|
135
|
+
finally:
|
|
136
|
+
stop.set()
|
|
137
|
+
t.join()
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@pytest.mark.parametrize("cls", TYPES)
|
|
141
|
+
def test_init_cannot_be_called_twice(cls):
|
|
142
|
+
m = cls({"a": 1})
|
|
143
|
+
with pytest.raises(TypeError, match="immutable"):
|
|
144
|
+
m.__init__({"b": 2})
|
|
145
|
+
assert m["a"] == 1
|
|
146
|
+
loaded = cls.from_bytes(m.to_bytes())
|
|
147
|
+
with pytest.raises(TypeError, match="immutable"):
|
|
148
|
+
loaded.__init__({"b": 2})
|
|
149
|
+
assert loaded["a"] == 1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|