fastconstmap 0.10.0__tar.gz → 0.11.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/PKG-INFO +24 -2
  2. fastconstmap-0.10.0/fastconstmap.egg-info/PKG-INFO → fastconstmap-0.11.0/README.md +22 -19
  3. fastconstmap-0.10.0/README.md → fastconstmap-0.11.0/fastconstmap.egg-info/PKG-INFO +41 -1
  4. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/fastconstmap.egg-info/SOURCES.txt +2 -1
  5. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/pyproject.toml +4 -1
  6. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/src/fastconstmap_module.c +122 -18
  7. fastconstmap-0.11.0/tests/test_threading.py +149 -0
  8. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/LICENSE +0 -0
  9. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/MANIFEST.in +0 -0
  10. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/benchmarks/bench_batch.c +0 -0
  11. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/benchmarks/benchmark.py +0 -0
  12. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/examples/example.py +0 -0
  13. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/examples/shared_memory_example.py +0 -0
  14. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/fastconstmap/__init__.py +0 -0
  15. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/fastconstmap/_fastconstmap.pyi +0 -0
  16. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/fastconstmap/py.typed +0 -0
  17. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/fastconstmap.egg-info/dependency_links.txt +0 -0
  18. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/fastconstmap.egg-info/top_level.txt +0 -0
  19. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/setup.cfg +0 -0
  20. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/setup.py +0 -0
  21. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/src/constmap.c +0 -0
  22. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/src/constmap.h +0 -0
  23. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/src/third_party/xxhash/LICENSE +0 -0
  24. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/src/third_party/xxhash/xxhash.h +0 -0
  25. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/tests/test_constmap.py +0 -0
  26. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/tests/test_interop.py +0 -0
  27. {fastconstmap-0.10.0 → fastconstmap-0.11.0}/tests/test_shared_memory.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: fastconstmap
3
- Version: 0.10.0
3
+ Version: 0.11.0
4
4
  Summary: Fast, immutable, compact map from strings to uint64 (binary fuse filter).
5
5
  Author-email: Daniel Lemire <daniel@lemire.me>
6
6
  License: Apache-2.0
@@ -8,6 +8,7 @@ Project-URL: Homepage, https://github.com/lemire/fastconstmap
8
8
  Project-URL: Source, https://github.com/lemire/fastconstmap
9
9
  Classifier: License :: OSI Approved :: Apache Software License
10
10
  Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable
11
12
  Classifier: Programming Language :: C
12
13
  Classifier: Operating System :: OS Independent
13
14
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
@@ -297,7 +298,7 @@ Notes and constraints:
297
298
  use. Drop the map (`del cm`) before calling `shm.close()`.
298
299
  - The map is **immutable** — the intended pattern is *write once in the
299
300
  producer, then only read in every process*. Concurrent readers need no
300
- locking.
301
+ locking (see [Threads](#threads-and-free-threaded-python)).
301
302
  - `from_buffer` verifies the magic bytes and the FNV-1a checksum, so a
302
303
  truncated or corrupt block raises `ValueError` rather than returning
303
304
  garbage.
@@ -313,6 +314,27 @@ Notes and constraints:
313
314
  load on a big-endian host): it copies the data and the resulting map owns
314
315
  its memory independently of the source buffer.
315
316
 
317
+ ## Threads and free-threaded Python
318
+
319
+ The maps are immutable, so any number of threads can look up keys in the
320
+ same map at once, without locking. The extension supports free-threaded
321
+ Python (3.13t, 3.14t): it does not re-enable the GIL on import, and lookups
322
+ from several threads then run in parallel.
323
+
324
+ ```python
325
+ from concurrent.futures import ThreadPoolExecutor
326
+
327
+ m = ConstMap(d)
328
+ with ThreadPoolExecutor(8) as pool:
329
+ results = list(pool.map(m.get_many, batches))
330
+ ```
331
+
332
+ Building a map from a dict that another thread is modifying is safe (on a
333
+ free-threaded build the map is built from a snapshot of the dict), and so is
334
+ a batched lookup over a list that another thread is modifying. `__init__` can only be called once on a
335
+ map; calling it again raises `TypeError` rather than replacing the data
336
+ under other readers.
337
+
316
338
  ## Interoperability with the Go and Rust implementations
317
339
 
318
340
  A map saved by any of [`constmap`][constmap] (Go), [`rsconstmap`][rsconstmap]
@@ -1,21 +1,3 @@
1
- Metadata-Version: 2.4
2
- Name: fastconstmap
3
- Version: 0.10.0
4
- Summary: Fast, immutable, compact map from strings to uint64 (binary fuse filter).
5
- Author-email: Daniel Lemire <daniel@lemire.me>
6
- License: Apache-2.0
7
- Project-URL: Homepage, https://github.com/lemire/fastconstmap
8
- Project-URL: Source, https://github.com/lemire/fastconstmap
9
- Classifier: License :: OSI Approved :: Apache Software License
10
- Classifier: Programming Language :: Python :: 3
11
- Classifier: Programming Language :: C
12
- Classifier: Operating System :: OS Independent
13
- Classifier: Topic :: Software Development :: Libraries :: Python Modules
14
- Requires-Python: >=3.8
15
- Description-Content-Type: text/markdown
16
- License-File: LICENSE
17
- Dynamic: license-file
18
-
19
1
  # fastconstmap
20
2
 
21
3
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE)
@@ -297,7 +279,7 @@ Notes and constraints:
297
279
  use. Drop the map (`del cm`) before calling `shm.close()`.
298
280
  - The map is **immutable** — the intended pattern is *write once in the
299
281
  producer, then only read in every process*. Concurrent readers need no
300
- locking.
282
+ locking (see [Threads](#threads-and-free-threaded-python)).
301
283
  - `from_buffer` verifies the magic bytes and the FNV-1a checksum, so a
302
284
  truncated or corrupt block raises `ValueError` rather than returning
303
285
  garbage.
@@ -313,6 +295,27 @@ Notes and constraints:
313
295
  load on a big-endian host): it copies the data and the resulting map owns
314
296
  its memory independently of the source buffer.
315
297
 
298
+ ## Threads and free-threaded Python
299
+
300
+ The maps are immutable, so any number of threads can look up keys in the
301
+ same map at once, without locking. The extension supports free-threaded
302
+ Python (3.13t, 3.14t): it does not re-enable the GIL on import, and lookups
303
+ from several threads then run in parallel.
304
+
305
+ ```python
306
+ from concurrent.futures import ThreadPoolExecutor
307
+
308
+ m = ConstMap(d)
309
+ with ThreadPoolExecutor(8) as pool:
310
+ results = list(pool.map(m.get_many, batches))
311
+ ```
312
+
313
+ Building a map from a dict that another thread is modifying is safe (on a
314
+ free-threaded build the map is built from a snapshot of the dict), and so is
315
+ a batched lookup over a list that another thread is modifying. `__init__` can only be called once on a
316
+ map; calling it again raises `TypeError` rather than replacing the data
317
+ under other readers.
318
+
316
319
  ## Interoperability with the Go and Rust implementations
317
320
 
318
321
  A map saved by any of [`constmap`][constmap] (Go), [`rsconstmap`][rsconstmap]
@@ -1,3 +1,22 @@
1
+ Metadata-Version: 2.4
2
+ Name: fastconstmap
3
+ Version: 0.11.0
4
+ Summary: Fast, immutable, compact map from strings to uint64 (binary fuse filter).
5
+ Author-email: Daniel Lemire <daniel@lemire.me>
6
+ License: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/lemire/fastconstmap
8
+ Project-URL: Source, https://github.com/lemire/fastconstmap
9
+ Classifier: License :: OSI Approved :: Apache Software License
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: Free Threading :: 3 - Stable
12
+ Classifier: Programming Language :: C
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
15
+ Requires-Python: >=3.8
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE
18
+ Dynamic: license-file
19
+
1
20
  # fastconstmap
2
21
 
3
22
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE)
@@ -279,7 +298,7 @@ Notes and constraints:
279
298
  use. Drop the map (`del cm`) before calling `shm.close()`.
280
299
  - The map is **immutable** — the intended pattern is *write once in the
281
300
  producer, then only read in every process*. Concurrent readers need no
282
- locking.
301
+ locking (see [Threads](#threads-and-free-threaded-python)).
283
302
  - `from_buffer` verifies the magic bytes and the FNV-1a checksum, so a
284
303
  truncated or corrupt block raises `ValueError` rather than returning
285
304
  garbage.
@@ -295,6 +314,27 @@ Notes and constraints:
295
314
  load on a big-endian host): it copies the data and the resulting map owns
296
315
  its memory independently of the source buffer.
297
316
 
317
+ ## Threads and free-threaded Python
318
+
319
+ The maps are immutable, so any number of threads can look up keys in the
320
+ same map at once, without locking. The extension supports free-threaded
321
+ Python (3.13t, 3.14t): it does not re-enable the GIL on import, and lookups
322
+ from several threads then run in parallel.
323
+
324
+ ```python
325
+ from concurrent.futures import ThreadPoolExecutor
326
+
327
+ m = ConstMap(d)
328
+ with ThreadPoolExecutor(8) as pool:
329
+ results = list(pool.map(m.get_many, batches))
330
+ ```
331
+
332
+ Building a map from a dict that another thread is modifying is safe (on a
333
+ free-threaded build the map is built from a snapshot of the dict), and so is
334
+ a batched lookup over a list that another thread is modifying. `__init__` can only be called once on a
335
+ map; calling it again raises `TypeError` rather than replacing the data
336
+ under other readers.
337
+
298
338
  ## Interoperability with the Go and Rust implementations
299
339
 
300
340
  A map saved by any of [`constmap`][constmap] (Go), [`rsconstmap`][rsconstmap]
@@ -21,4 +21,5 @@ src/third_party/xxhash/LICENSE
21
21
  src/third_party/xxhash/xxhash.h
22
22
  tests/test_constmap.py
23
23
  tests/test_interop.py
24
- tests/test_shared_memory.py
24
+ tests/test_shared_memory.py
25
+ tests/test_threading.py
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "fastconstmap"
7
- version = "0.10.0"
7
+ version = "0.11.0"
8
8
  description = "Fast, immutable, compact map from strings to uint64 (binary fuse filter)."
9
9
  readme = "README.md"
10
10
  license = { text = "Apache-2.0" }
@@ -13,6 +13,7 @@ requires-python = ">=3.8"
13
13
  classifiers = [
14
14
  "License :: OSI Approved :: Apache Software License",
15
15
  "Programming Language :: Python :: 3",
16
+ "Programming Language :: Python :: Free Threading :: 3 - Stable",
16
17
  "Programming Language :: C",
17
18
  "Operating System :: OS Independent",
18
19
  "Topic :: Software Development :: Libraries :: Python Modules",
@@ -33,6 +34,8 @@ fastconstmap = ["py.typed"]
33
34
  # our pytest suite — catches packaging regressions before they ship.
34
35
  test-requires = "pytest"
35
36
  test-command = "pytest {project}/tests"
37
+ # Also build the free-threaded 3.13t wheels (3.14t is built by default).
38
+ enable = ["cpython-freethreading"]
36
39
 
37
40
  [tool.cibuildwheel.macos]
38
41
  archs = ["x86_64", "universal2", "arm64"]
@@ -13,6 +13,35 @@
13
13
  #include <stdint.h>
14
14
  #include <string.h>
15
15
 
16
+ /* ------------------------------------------------------------------------- */
17
+ /* Free threading */
18
+ /* ------------------------------------------------------------------------- */
19
+
20
+ /* The maps are immutable once built, so lookups need no locking. What needs
21
+ * care on a free-threaded build is (1) the dict and lists we read, which
22
+ * another thread may be mutating, and (2) __init__, which installs the map.
23
+ * Critical sections only exist on 3.13+; with the GIL they are not needed. */
24
+ #ifndef Py_BEGIN_CRITICAL_SECTION
25
+ #define Py_BEGIN_CRITICAL_SECTION(op) {
26
+ #define Py_END_CRITICAL_SECTION() }
27
+ #endif
28
+
29
+ /* Like PySequence_Fast, but on a free-threaded build a list (which another
30
+ * thread could mutate while we read its items) is snapshotted into a tuple.
31
+ * Any other iterable already becomes a fresh, private list. */
32
+ static PyObject *fcm_sequence_fast(PyObject *obj, const char *msg) {
33
+ #ifdef Py_GIL_DISABLED
34
+ if (PyList_Check(obj)) return PyList_AsTuple(obj);
35
+ #endif
36
+ return PySequence_Fast(obj, msg);
37
+ }
38
+
39
+ static int fcm_already_initialized(const char *type_name) {
40
+ PyErr_Format(PyExc_TypeError,
41
+ "%s is immutable: __init__ cannot be called again", type_name);
42
+ return -1;
43
+ }
44
+
16
45
  /* ------------------------------------------------------------------------- */
17
46
  /* Batched-lookup helpers */
18
47
  /* ------------------------------------------------------------------------- */
@@ -134,12 +163,17 @@ static int fcm_dict_to_arrays(PyObject *dict,
134
163
  PyErr_SetString(PyExc_TypeError, "expected a dict");
135
164
  return -1;
136
165
  }
137
- Py_ssize_t n = PyDict_Size(dict);
138
- if (n < 0) return -1;
139
-
140
166
  fcm_key_t *keys = NULL;
141
167
  uint64_t *vals = NULL;
142
168
  PyObject *keep = NULL;
169
+ #ifdef Py_GIL_DISABLED
170
+ /* Iterate over a private copy: another thread may mutate `dict`. */
171
+ PyObject *copy = PyDict_Copy(dict);
172
+ if (!copy) return -1;
173
+ dict = copy;
174
+ #endif
175
+ Py_ssize_t n = PyDict_Size(dict);
176
+ if (n < 0) goto fail;
143
177
 
144
178
  keep = PyTuple_New(n);
145
179
  if (!keep) goto fail;
@@ -205,12 +239,18 @@ static int fcm_dict_to_arrays(PyObject *dict,
205
239
  *out_keys = keys;
206
240
  *out_values = vals;
207
241
  *out_keep = keep;
242
+ #ifdef Py_GIL_DISABLED
243
+ Py_DECREF(copy);
244
+ #endif
208
245
  return 0;
209
246
 
210
247
  fail:
211
248
  PyMem_Free(keys);
212
249
  PyMem_Free(vals);
213
250
  Py_XDECREF(keep);
251
+ #ifdef Py_GIL_DISABLED
252
+ Py_DECREF(copy);
253
+ #endif
214
254
  return -1;
215
255
  }
216
256
 
@@ -224,6 +264,8 @@ typedef struct {
224
264
  /* When `view.obj` is non-NULL the map is a zero-copy view: `cm.data`
225
265
  * points into `view`'s buffer and must NOT be freed. */
226
266
  Py_buffer view;
267
+ /* Set once the map holds its data; it never changes afterwards. */
268
+ int initialized;
227
269
  } PyConstMap;
228
270
 
229
271
  /* Release whatever storage the map currently holds (owned heap data or a
@@ -251,11 +293,13 @@ static int PyConstMap_init(PyConstMap *self, PyObject *args, PyObject *kwargs) {
251
293
  PyObject *keep = NULL;
252
294
  if (fcm_dict_to_arrays(dict, &n, &keys, &vals, &keep) < 0) return -1;
253
295
 
254
- pyconstmap_release(self);
255
-
296
+ /* Build into a local map and install it only if the object holds none:
297
+ * replacing a map would free memory other threads may be reading. */
298
+ fcm_constmap_t built;
299
+ memset(&built, 0, sizeof(built));
256
300
  int rc;
257
301
  Py_BEGIN_ALLOW_THREADS
258
- rc = fcm_constmap_new_with_hash(&self->cm, keys, vals, n, hash);
302
+ rc = fcm_constmap_new_with_hash(&built, keys, vals, n, hash);
259
303
  Py_END_ALLOW_THREADS
260
304
 
261
305
  PyMem_Free(keys);
@@ -263,6 +307,18 @@ static int PyConstMap_init(PyConstMap *self, PyObject *args, PyObject *kwargs) {
263
307
  Py_DECREF(keep);
264
308
 
265
309
  if (rc != FCM_OK) { fcm_set_python_error(rc); return -1; }
310
+ int installed = 0;
311
+ Py_BEGIN_CRITICAL_SECTION(self);
312
+ if (!self->initialized) {
313
+ self->cm = built;
314
+ self->initialized = 1;
315
+ installed = 1;
316
+ }
317
+ Py_END_CRITICAL_SECTION();
318
+ if (!installed) {
319
+ fcm_constmap_free(&built);
320
+ return fcm_already_initialized("ConstMap");
321
+ }
266
322
  return 0;
267
323
  }
268
324
 
@@ -295,7 +351,7 @@ static PyObject *PyConstMap_get(PyConstMap *self, PyObject *arg) {
295
351
  * gain over a loop of single lookups comes from once the map no longer fits in
296
352
  * cache. */
297
353
  static PyObject *PyConstMap_get_many(PyConstMap *self, PyObject *arg) {
298
- PyObject *seq = PySequence_Fast(arg, "get_many() requires an iterable of keys");
354
+ PyObject *seq = fcm_sequence_fast(arg, "get_many() requires an iterable of keys");
299
355
  if (!seq) return NULL;
300
356
  Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
301
357
  PyObject **items = PySequence_Fast_ITEMS(seq);
@@ -332,7 +388,7 @@ static PyObject *PyConstMap_get_many_into(PyConstMap *self, PyObject *args) {
332
388
  PyObject *keys_arg, *out_arg;
333
389
  if (!PyArg_ParseTuple(args, "OO:get_many_into", &keys_arg, &out_arg)) return NULL;
334
390
 
335
- PyObject *seq = PySequence_Fast(keys_arg, "get_many_into() requires an iterable of keys");
391
+ PyObject *seq = fcm_sequence_fast(keys_arg, "get_many_into() requires an iterable of keys");
336
392
  if (!seq) return NULL;
337
393
  Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
338
394
  PyObject **items = PySequence_Fast_ITEMS(seq);
@@ -397,6 +453,7 @@ static PyObject *PyConstMap_deserialize(PyTypeObject *type, PyObject *arg) {
397
453
  fcm_set_python_error(rc);
398
454
  return NULL;
399
455
  }
456
+ self->initialized = 1;
400
457
  return (PyObject *)self;
401
458
  }
402
459
 
@@ -446,6 +503,7 @@ static PyObject *PyConstMap_load(PyTypeObject *type, PyObject *arg) {
446
503
  int rc = fcm_constmap_read(&self->cm, buf, (size_t)len);
447
504
  PyMem_Free(buf);
448
505
  if (rc != FCM_OK) { Py_DECREF(self); fcm_set_python_error(rc); return NULL; }
506
+ self->initialized = 1;
449
507
  return (PyObject *)self;
450
508
  }
451
509
 
@@ -497,6 +555,7 @@ static PyObject *PyConstMap_from_buffer(PyTypeObject *type, PyObject *arg) {
497
555
  return NULL;
498
556
  }
499
557
  self->view = view; /* keep the buffer alive for the map's lifetime */
558
+ self->initialized = 1;
500
559
  return (PyObject *)self;
501
560
  }
502
561
 
@@ -571,6 +630,8 @@ typedef struct {
571
630
  fcm_verified_constmap_t vm;
572
631
  /* Non-NULL view.obj => zero-copy view: data/checks point into the buffer. */
573
632
  Py_buffer view;
633
+ /* Set once the map holds its data; it never changes afterwards. */
634
+ int initialized;
574
635
  } PyVerifiedConstMap;
575
636
 
576
637
  static void pyverified_release(PyVerifiedConstMap *self) {
@@ -596,11 +657,13 @@ static int PyVerifiedConstMap_init(PyVerifiedConstMap *self, PyObject *args, PyO
596
657
  PyObject *keep = NULL;
597
658
  if (fcm_dict_to_arrays(dict, &n, &keys, &vals, &keep) < 0) return -1;
598
659
 
599
- pyverified_release(self);
600
-
660
+ /* Build into a local map and install it only if the object holds none:
661
+ * replacing a map would free memory other threads may be reading. */
662
+ fcm_verified_constmap_t built;
663
+ memset(&built, 0, sizeof(built));
601
664
  int rc;
602
665
  Py_BEGIN_ALLOW_THREADS
603
- rc = fcm_verified_constmap_new_with_hash(&self->vm, keys, vals, n, hash);
666
+ rc = fcm_verified_constmap_new_with_hash(&built, keys, vals, n, hash);
604
667
  Py_END_ALLOW_THREADS
605
668
 
606
669
  PyMem_Free(keys);
@@ -608,6 +671,18 @@ static int PyVerifiedConstMap_init(PyVerifiedConstMap *self, PyObject *args, PyO
608
671
  Py_DECREF(keep);
609
672
 
610
673
  if (rc != FCM_OK) { fcm_set_python_error(rc); return -1; }
674
+ int installed = 0;
675
+ Py_BEGIN_CRITICAL_SECTION(self);
676
+ if (!self->initialized) {
677
+ self->vm = built;
678
+ self->initialized = 1;
679
+ installed = 1;
680
+ }
681
+ Py_END_CRITICAL_SECTION();
682
+ if (!installed) {
683
+ fcm_verified_constmap_free(&built);
684
+ return fcm_already_initialized("VerifiedConstMap");
685
+ }
611
686
  return 0;
612
687
  }
613
688
 
@@ -680,7 +755,7 @@ static PyObject *PyVerifiedConstMap_get_many(PyVerifiedConstMap *self, PyObject
680
755
  static char *kwlist[] = {"keys", "default", NULL};
681
756
  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", kwlist, &arg, &default_)) return NULL;
682
757
 
683
- PyObject *seq = PySequence_Fast(arg, "get_many() requires an iterable of keys");
758
+ PyObject *seq = fcm_sequence_fast(arg, "get_many() requires an iterable of keys");
684
759
  if (!seq) return NULL;
685
760
  Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
686
761
  PyObject **items = PySequence_Fast_ITEMS(seq);
@@ -723,7 +798,7 @@ static PyObject *PyVerifiedConstMap_get_many_into(PyVerifiedConstMap *self, PyOb
723
798
  PyObject *keys_arg, *out_arg;
724
799
  if (!PyArg_ParseTuple(args, "OO:get_many_into", &keys_arg, &out_arg)) return NULL;
725
800
 
726
- PyObject *seq = PySequence_Fast(keys_arg, "get_many_into() requires an iterable of keys");
801
+ PyObject *seq = fcm_sequence_fast(keys_arg, "get_many_into() requires an iterable of keys");
727
802
  if (!seq) return NULL;
728
803
  Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
729
804
  PyObject **items = PySequence_Fast_ITEMS(seq);
@@ -778,6 +853,7 @@ static PyObject *PyVerifiedConstMap_deserialize(PyTypeObject *type, PyObject *ar
778
853
  int rc = fcm_verified_constmap_read(&self->vm, view.buf, (size_t)view.len);
779
854
  PyBuffer_Release(&view);
780
855
  if (rc != FCM_OK) { Py_DECREF(self); fcm_set_python_error(rc); return NULL; }
856
+ self->initialized = 1;
781
857
  return (PyObject *)self;
782
858
  }
783
859
 
@@ -824,6 +900,7 @@ static PyObject *PyVerifiedConstMap_load(PyTypeObject *type, PyObject *arg) {
824
900
  int rc = fcm_verified_constmap_read(&self->vm, buf, (size_t)len);
825
901
  PyMem_Free(buf);
826
902
  if (rc != FCM_OK) { Py_DECREF(self); fcm_set_python_error(rc); return NULL; }
903
+ self->initialized = 1;
827
904
  return (PyObject *)self;
828
905
  }
829
906
 
@@ -870,6 +947,7 @@ static PyObject *PyVerifiedConstMap_from_buffer(PyTypeObject *type, PyObject *ar
870
947
  return NULL;
871
948
  }
872
949
  self->view = view;
950
+ self->initialized = 1;
873
951
  return (PyObject *)self;
874
952
  }
875
953
 
@@ -948,6 +1026,8 @@ typedef struct {
948
1026
  fcm_paired_verified_constmap_t vm;
949
1027
  /* Non-NULL view.obj => zero-copy view: slots point into the buffer. */
950
1028
  Py_buffer view;
1029
+ /* Set once the map holds its data; it never changes afterwards. */
1030
+ int initialized;
951
1031
  } PyPairedVerifiedConstMap;
952
1032
 
953
1033
  static void pypaired_release(PyPairedVerifiedConstMap *self) {
@@ -972,11 +1052,13 @@ static int PyPairedVerifiedConstMap_init(PyPairedVerifiedConstMap *self, PyObjec
972
1052
  PyObject *keep = NULL;
973
1053
  if (fcm_dict_to_arrays(dict, &n, &keys, &vals, &keep) < 0) return -1;
974
1054
 
975
- pypaired_release(self);
976
-
1055
+ /* Build into a local map and install it only if the object holds none:
1056
+ * replacing a map would free memory other threads may be reading. */
1057
+ fcm_paired_verified_constmap_t built;
1058
+ memset(&built, 0, sizeof(built));
977
1059
  int rc;
978
1060
  Py_BEGIN_ALLOW_THREADS
979
- rc = fcm_paired_verified_constmap_new_with_hash(&self->vm, keys, vals, n, hash);
1061
+ rc = fcm_paired_verified_constmap_new_with_hash(&built, keys, vals, n, hash);
980
1062
  Py_END_ALLOW_THREADS
981
1063
 
982
1064
  PyMem_Free(keys);
@@ -984,6 +1066,18 @@ static int PyPairedVerifiedConstMap_init(PyPairedVerifiedConstMap *self, PyObjec
984
1066
  Py_DECREF(keep);
985
1067
 
986
1068
  if (rc != FCM_OK) { fcm_set_python_error(rc); return -1; }
1069
+ int installed = 0;
1070
+ Py_BEGIN_CRITICAL_SECTION(self);
1071
+ if (!self->initialized) {
1072
+ self->vm = built;
1073
+ self->initialized = 1;
1074
+ installed = 1;
1075
+ }
1076
+ Py_END_CRITICAL_SECTION();
1077
+ if (!installed) {
1078
+ fcm_paired_verified_constmap_free(&built);
1079
+ return fcm_already_initialized("PairedVerifiedConstMap");
1080
+ }
987
1081
  return 0;
988
1082
  }
989
1083
 
@@ -1056,7 +1150,7 @@ static PyObject *PyPairedVerifiedConstMap_get_many(PyPairedVerifiedConstMap *sel
1056
1150
  static char *kwlist[] = {"keys", "default", NULL};
1057
1151
  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", kwlist, &arg, &default_)) return NULL;
1058
1152
 
1059
- PyObject *seq = PySequence_Fast(arg, "get_many() requires an iterable of keys");
1153
+ PyObject *seq = fcm_sequence_fast(arg, "get_many() requires an iterable of keys");
1060
1154
  if (!seq) return NULL;
1061
1155
  Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
1062
1156
  PyObject **items = PySequence_Fast_ITEMS(seq);
@@ -1099,7 +1193,7 @@ static PyObject *PyPairedVerifiedConstMap_get_many_into(PyPairedVerifiedConstMap
1099
1193
  PyObject *keys_arg, *out_arg;
1100
1194
  if (!PyArg_ParseTuple(args, "OO:get_many_into", &keys_arg, &out_arg)) return NULL;
1101
1195
 
1102
- PyObject *seq = PySequence_Fast(keys_arg, "get_many_into() requires an iterable of keys");
1196
+ PyObject *seq = fcm_sequence_fast(keys_arg, "get_many_into() requires an iterable of keys");
1103
1197
  if (!seq) return NULL;
1104
1198
  Py_ssize_t n = PySequence_Fast_GET_SIZE(seq);
1105
1199
  PyObject **items = PySequence_Fast_ITEMS(seq);
@@ -1154,6 +1248,7 @@ static PyObject *PyPairedVerifiedConstMap_deserialize(PyTypeObject *type, PyObje
1154
1248
  int rc = fcm_paired_verified_constmap_read(&self->vm, view.buf, (size_t)view.len);
1155
1249
  PyBuffer_Release(&view);
1156
1250
  if (rc != FCM_OK) { Py_DECREF(self); fcm_set_python_error(rc); return NULL; }
1251
+ self->initialized = 1;
1157
1252
  return (PyObject *)self;
1158
1253
  }
1159
1254
 
@@ -1200,6 +1295,7 @@ static PyObject *PyPairedVerifiedConstMap_load(PyTypeObject *type, PyObject *arg
1200
1295
  int rc = fcm_paired_verified_constmap_read(&self->vm, buf, (size_t)len);
1201
1296
  PyMem_Free(buf);
1202
1297
  if (rc != FCM_OK) { Py_DECREF(self); fcm_set_python_error(rc); return NULL; }
1298
+ self->initialized = 1;
1203
1299
  return (PyObject *)self;
1204
1300
  }
1205
1301
 
@@ -1246,6 +1342,7 @@ static PyObject *PyPairedVerifiedConstMap_from_buffer(PyTypeObject *type, PyObje
1246
1342
  return NULL;
1247
1343
  }
1248
1344
  self->view = view;
1345
+ self->initialized = 1;
1249
1346
  return (PyObject *)self;
1250
1347
  }
1251
1348
 
@@ -1334,6 +1431,13 @@ PyMODINIT_FUNC PyInit__fastconstmap(void) {
1334
1431
 
1335
1432
  PyObject *m = PyModule_Create(&fastconstmap_module);
1336
1433
  if (!m) return NULL;
1434
+ #ifdef Py_GIL_DISABLED
1435
+ /* Safe to run without the GIL (see "Free threading" above). */
1436
+ if (PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED) < 0) {
1437
+ Py_DECREF(m);
1438
+ return NULL;
1439
+ }
1440
+ #endif
1337
1441
 
1338
1442
  Py_INCREF(&PyConstMapType);
1339
1443
  if (PyModule_AddObject(m, "ConstMap", (PyObject *)&PyConstMapType) < 0) {
@@ -0,0 +1,149 @@
1
+ """Concurrent use of the maps. On a free-threaded build (python3.14t) these
2
+ run truly in parallel; with the GIL they still exercise the same code."""
3
+ import array
4
+ import sys
5
+ import sysconfig
6
+ import threading
7
+
8
+ import pytest
9
+
10
+ from fastconstmap import NOT_FOUND, ConstMap, PairedVerifiedConstMap, VerifiedConstMap
11
+
12
+ N_THREADS = 8
13
+ TYPES = [ConstMap, VerifiedConstMap, PairedVerifiedConstMap]
14
+
15
+
16
+ def run_threads(target, n=N_THREADS):
17
+ barrier = threading.Barrier(n)
18
+ errors = []
19
+
20
+ def wrapper(i):
21
+ barrier.wait()
22
+ try:
23
+ target(i)
24
+ except BaseException as e: # pragma: no cover - reported below
25
+ errors.append(e)
26
+
27
+ threads = [threading.Thread(target=wrapper, args=(i,)) for i in range(n)]
28
+ for t in threads:
29
+ t.start()
30
+ for t in threads:
31
+ t.join()
32
+ if errors:
33
+ raise errors[0]
34
+
35
+
36
+ def make_data(n=20_000):
37
+ return {f"key-{i}": i * 7 for i in range(n)}
38
+
39
+
40
+ @pytest.mark.skipif(not sysconfig.get_config_var("Py_GIL_DISABLED"),
41
+ reason="needs a free-threaded build")
42
+ def test_gil_stays_disabled():
43
+ # Importing the extension must not re-enable the GIL.
44
+ assert not sys._is_gil_enabled()
45
+
46
+
47
+ @pytest.mark.parametrize("cls", TYPES)
48
+ def test_concurrent_lookups(cls):
49
+ data = make_data()
50
+ m = cls(data)
51
+ keys = list(data)
52
+ expected = [data[k] for k in keys]
53
+
54
+ def work(i):
55
+ for _ in range(3):
56
+ assert m.get_many(keys) == expected
57
+ out = array.array("Q", bytes(8 * len(keys)))
58
+ assert m.get_many_into(keys, out) == len(keys)
59
+ assert list(out) == expected
60
+ for k in keys[i::N_THREADS]:
61
+ assert m[k] == data[k]
62
+
63
+ run_threads(work)
64
+
65
+
66
+ @pytest.mark.parametrize("cls", [VerifiedConstMap, PairedVerifiedConstMap])
67
+ def test_concurrent_absent_keys(cls):
68
+ m = cls(make_data())
69
+ absent = [f"absent-{i}" for i in range(5_000)]
70
+
71
+ def work(i):
72
+ assert m.get_many(absent) == [None] * len(absent)
73
+ out = array.array("Q", bytes(8 * len(absent)))
74
+ m.get_many_into(absent, out)
75
+ assert set(out) == {NOT_FOUND}
76
+
77
+ run_threads(work)
78
+
79
+
80
+ @pytest.mark.parametrize("cls", TYPES)
81
+ def test_concurrent_construction(cls):
82
+ data = make_data(5_000)
83
+ blob = cls(data).to_bytes()
84
+
85
+ def work(i):
86
+ local = {f"{i}-{k}": v for k, v in data.items()}
87
+ m = cls(local)
88
+ assert m.get_many(list(local)) == list(local.values())
89
+ assert cls.from_bytes(blob).to_bytes() == blob
90
+
91
+ run_threads(work)
92
+
93
+
94
+ @pytest.mark.parametrize("cls", TYPES)
95
+ def test_lookup_while_list_mutates(cls):
96
+ # A key list shared with a thread that mutates it must not crash the
97
+ # lookup; the results are whatever keys the list held at the time.
98
+ data = make_data(2_000)
99
+ m = cls(data)
100
+ keys = list(data)
101
+ stop = threading.Event()
102
+
103
+ def mutate():
104
+ while not stop.is_set():
105
+ keys.append("x")
106
+ keys.pop()
107
+ keys.reverse()
108
+
109
+ t = threading.Thread(target=mutate)
110
+ t.start()
111
+ try:
112
+ for _ in range(200):
113
+ assert len(m.get_many(keys)) in (len(data), len(data) + 1)
114
+ finally:
115
+ stop.set()
116
+ t.join()
117
+
118
+
119
+ @pytest.mark.parametrize("cls", TYPES)
120
+ def test_construct_while_dict_mutates(cls):
121
+ data = make_data(2_000)
122
+ stop = threading.Event()
123
+
124
+ def mutate():
125
+ while not stop.is_set():
126
+ data["extra"] = 1
127
+ del data["extra"]
128
+
129
+ t = threading.Thread(target=mutate)
130
+ t.start()
131
+ try:
132
+ for _ in range(50):
133
+ m = cls(data)
134
+ assert m["key-0"] == 0
135
+ finally:
136
+ stop.set()
137
+ t.join()
138
+
139
+
140
+ @pytest.mark.parametrize("cls", TYPES)
141
+ def test_init_cannot_be_called_twice(cls):
142
+ m = cls({"a": 1})
143
+ with pytest.raises(TypeError, match="immutable"):
144
+ m.__init__({"b": 2})
145
+ assert m["a"] == 1
146
+ loaded = cls.from_bytes(m.to_bytes())
147
+ with pytest.raises(TypeError, match="immutable"):
148
+ loaded.__init__({"b": 2})
149
+ assert loaded["a"] == 1
File without changes
File without changes
File without changes
File without changes