py-sharedmemory 4.1.1__tar.gz → 4.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/CHANGELOG.md +22 -0
- {py_sharedmemory-4.1.1/py_sharedmemory.egg-info → py_sharedmemory-4.2.0}/PKG-INFO +35 -21
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/README.md +34 -20
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/py_sharedmemory/__init__.py +1 -1
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/py_sharedmemory/convert.py +53 -21
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/py_sharedmemory/receiver.py +10 -8
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/py_sharedmemory/sender.py +104 -29
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0/py_sharedmemory.egg-info}/PKG-INFO +35 -21
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/py_sharedmemory.egg-info/SOURCES.txt +1 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/tests/conftest.py +2 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/tests/helpers.py +11 -1
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/tests/test_convert.py +56 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/tests/test_queue.py +108 -33
- py_sharedmemory-4.2.0/tests/test_recycling.py +253 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/tests/test_threads.py +4 -4
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/LICENSE +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/MANIFEST.in +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/benchmarks/__init__.py +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/benchmarks/benchmark.py +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/benchmarks/plot.py +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/benchmarks/results/windows-numpy.json +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/benchmarks/results/windows-numpy.png +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/examples/basic.py +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/examples/numpy_batches.py +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/examples/worker_pool.py +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/py_sharedmemory/factory.py +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/py_sharedmemory/py.typed +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/py_sharedmemory.egg-info/dependency_links.txt +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/py_sharedmemory.egg-info/requires.txt +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/py_sharedmemory.egg-info/top_level.txt +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/pyproject.toml +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/setup.cfg +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/tests/__init__.py +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/tests/test_benchmark.py +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/tests/test_objects.py +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/tests/test_package.py +0 -0
- {py_sharedmemory-4.1.1 → py_sharedmemory-4.2.0}/tests/test_processes.py +0 -0
|
@@ -1,5 +1,27 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 4.2.0
|
|
4
|
+
|
|
5
|
+
- Reuse received shared-memory blocks instead of freeing them. Writing into fresh pages
|
|
6
|
+
cost far more than the copy itself, so large `put` calls are now much faster. A block
|
|
7
|
+
is reused for a message that fills at least half of it, or for any message that fits a
|
|
8
|
+
block of up to 1 MiB. The sender keeps at most 8 blocks and 256 MiB for reuse, never
|
|
9
|
+
more blocks than the capacity allows, and frees them on `close()`, so peak memory is
|
|
10
|
+
unchanged. On Linux and macOS at most 8 unread blocks stay mapped (two file
|
|
11
|
+
descriptors each).
|
|
12
|
+
- Acknowledgements carry a per-message sequence number, so an unknown or repeated
|
|
13
|
+
acknowledgement can never release a reused block that holds a newer message.
|
|
14
|
+
- Fix receivers hanging forever in `close()` or at process exit after reading many
|
|
15
|
+
messages that a closed sender had already freed: such messages are no longer
|
|
16
|
+
acknowledged, so the acknowledgement pipe can no longer fill up.
|
|
17
|
+
- Fix a Windows race where a message freed while a receiver attached to it raised a
|
|
18
|
+
confusing `UnpicklingError`; it now raises the usual "freed before it was received"
|
|
19
|
+
`FileNotFoundError`.
|
|
20
|
+
- Received NumPy arrays are always aligned. Arrays that followed an odd-sized buffer in the
|
|
21
|
+
same message used to arrive unaligned, which slows NumPy and breaks libraries that need
|
|
22
|
+
aligned memory. Each array now owns its memory, so keeping one no longer keeps the
|
|
23
|
+
whole message alive.
|
|
24
|
+
|
|
3
25
|
## 4.1.1
|
|
4
26
|
|
|
5
27
|
- Unread messages no longer hold file descriptors on Linux and macOS: the sender unmaps
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: py-sharedmemory
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.2.0
|
|
4
4
|
Summary: Fast multiprocessing.Queue alternative that moves large Python objects and NumPy arrays through shared memory
|
|
5
5
|
Author: Tobias Würth
|
|
6
6
|
License-Expression: MIT
|
|
@@ -70,8 +70,8 @@ images, and simulation states arrive up to 2× faster, and transfers too large f
|
|
|
70
70
|
`queue.Full`, and `queue.Empty`.
|
|
71
71
|
- **Any picklable object:** NumPy arrays and other buffers use pickle protocol 5
|
|
72
72
|
out-of-band buffers, so they are copied straight into shared memory.
|
|
73
|
-
- **Bounded memory:** a capacity limit applies backpressure, and
|
|
74
|
-
|
|
73
|
+
- **Bounded memory:** a capacity limit applies backpressure, and received blocks are
|
|
74
|
+
reused for later messages, so shared memory stays within `capacity × message size`.
|
|
75
75
|
- **One producer, many consumers:** consumer processes and threads share one receiver.
|
|
76
76
|
- **Portable:** Linux, macOS, and Windows; `spawn`, `fork`, and `forkserver`; Python 3.11–3.15.
|
|
77
77
|
- **Lightweight:** no dependencies, fully typed (PEP 561), and 100% test coverage.
|
|
@@ -131,8 +131,8 @@ arrays, image or video frames, RL rollouts, large `bytes`, or big nested objects
|
|
|
131
131
|
helps where large `multiprocessing.Queue` transfers cause heavy pipe I/O or uneven
|
|
132
132
|
throughput, which is especially noticeable on Windows.
|
|
133
133
|
|
|
134
|
-
Prefer `multiprocessing.Queue` for small messages. Below about 100 kB,
|
|
135
|
-
|
|
134
|
+
Prefer `multiprocessing.Queue` for small messages. Below about 100 kB, the fixed cost of
|
|
135
|
+
mapping a shared-memory block for every message outweighs the pipe.
|
|
136
136
|
|
|
137
137
|
To let several processes read and write **the same** array in place, use
|
|
138
138
|
[`multiprocessing.shared_memory`](https://docs.python.org/3/library/multiprocessing.shared_memory.html)
|
|
@@ -155,20 +155,24 @@ Core i9-14900K, 191.8 GiB RAM, Windows 11, Python 3.12.10, NumPy 2.5.3, measured
|
|
|
155
155
|
2026-10-08. Contiguous float32 arrays, `spawn`, capacity 2, median of 5 runs excluding
|
|
156
156
|
startup and warmup; up to 1,000 messages or 512 MiB per run, at least one message.
|
|
157
157
|
1 GB = 10⁹ bytes. See the [raw results](https://github.com/tobiaswuerth/python_shared_memory_queue/blob/main/benchmarks/results/windows-numpy.json).
|
|
158
|
-
These numbers show transfer speed; I/O load and stability depend on the workload.
|
|
158
|
+
These numbers show transfer speed; I/O load and stability depend on the workload. They
|
|
159
|
+
predate version 4.2, which reuses received blocks and so no longer pays to fault in fresh
|
|
160
|
+
pages on every `put`.
|
|
159
161
|
|
|
160
162
|
## How it works
|
|
161
163
|
|
|
162
164
|
1. `put()` pickles the object with protocol 5. Large buffers, such as NumPy array data,
|
|
163
|
-
stay out-of-band, and everything is copied into one
|
|
164
|
-
|
|
165
|
+
stay out-of-band, and everything is copied into one shared-memory block: a block from
|
|
166
|
+
an earlier, received message when one fits, otherwise a new one.
|
|
167
|
+
2. Only the block's name, buffer sizes, and a sequence number travel through a
|
|
168
|
+
`multiprocessing.Queue`.
|
|
165
169
|
3. `get()` attaches to the block, copies the data into local memory, unpickles it, and
|
|
166
170
|
sends an acknowledgement back.
|
|
167
|
-
4. A background thread in the producer
|
|
168
|
-
arrives. `capacity` limits how many
|
|
171
|
+
4. A background thread in the producer makes the block available for reuse when the
|
|
172
|
+
acknowledgement arrives. `capacity` limits how many messages can be outstanding.
|
|
169
173
|
|
|
170
174
|
Received objects never point into shared memory, so they stay valid after the block is
|
|
171
|
-
freed and can be modified freely.
|
|
175
|
+
reused or freed and can be modified freely.
|
|
172
176
|
|
|
173
177
|
## Usage guide
|
|
174
178
|
|
|
@@ -188,9 +192,15 @@ consumer = ctx.Process(target=consume, args=(receiver,))
|
|
|
188
192
|
|
|
189
193
|
### Capacity and backpressure
|
|
190
194
|
|
|
191
|
-
`capacity` counts messages that were sent but not yet received
|
|
192
|
-
|
|
193
|
-
|
|
195
|
+
`capacity` counts messages that were sent but not yet received. When it is reached,
|
|
196
|
+
`put()` blocks, so a fast producer cannot exhaust memory. Shared-memory use is at most
|
|
197
|
+
about `capacity × message size`. `capacity=0` or `None` means unbounded.
|
|
198
|
+
|
|
199
|
+
Received blocks are reused rather than freed, because writing into memory that is
|
|
200
|
+
already mapped is much faster than faulting in fresh pages. A block is reused for a
|
|
201
|
+
message that fills at least half of it, or for any message that fits a block of up to
|
|
202
|
+
1 MiB. The sender keeps at most 8 such blocks and 256 MiB, never more blocks than the
|
|
203
|
+
capacity allows, and frees them on `close()`. Blocks over 256 MiB are freed once received.
|
|
194
204
|
|
|
195
205
|
### Timeouts and non-blocking calls
|
|
196
206
|
|
|
@@ -241,7 +251,9 @@ such as `None` is a simple way to tell consumers that the stream has ended.
|
|
|
241
251
|
Arrays keep their dtype, shape, and writability. Read-only arrays stay read-only, and
|
|
242
252
|
writable ones arrive as writable local copies. C- and Fortran-contiguous arrays keep
|
|
243
253
|
their layout and are copied straight into shared memory without an intermediate pickle
|
|
244
|
-
copy; other views, such as strided slices, arrive as contiguous copies.
|
|
254
|
+
copy; other views, such as strided slices, arrive as contiguous copies. Each received
|
|
255
|
+
array has its own aligned memory, so keeping one array does not keep the rest of the
|
|
256
|
+
message alive.
|
|
245
257
|
|
|
246
258
|
### Errors
|
|
247
259
|
|
|
@@ -250,8 +262,9 @@ copy; other views, such as strided slices, arrive as contiguous copies.
|
|
|
250
262
|
| `queue.Full` | `put` found no capacity within `timeout` or with `block=False`. |
|
|
251
263
|
| `queue.Empty` | `get` found no message within `timeout` or with `block=False`. |
|
|
252
264
|
| `BrokenPipeError` | The endpoint is closed, or the producer's acknowledgement channel failed. |
|
|
265
|
+
| `FileNotFoundError` | `get` received a message that the producer freed by closing before it was read. |
|
|
253
266
|
| `RuntimeError` | A second process tries to produce, or a used sender is passed to another process. |
|
|
254
|
-
| Pickling errors | The object cannot be pickled or unpickled. Both endpoints stay usable, and failed messages are still
|
|
267
|
+
| Pickling errors | The object cannot be pickled or unpickled. Both endpoints stay usable, and failed messages are still released. |
|
|
255
268
|
| `RuntimeWarning` (warning) | A `with sender:` block or the producing process ended normally and discarded unacknowledged messages. |
|
|
256
269
|
|
|
257
270
|
## API reference
|
|
@@ -266,10 +279,10 @@ for the underlying queues; it defaults to `multiprocessing.get_context()`.
|
|
|
266
279
|
|
|
267
280
|
| Member | Description |
|
|
268
281
|
| --- | --- |
|
|
269
|
-
| `put(obj, block=True, timeout=None)` | Copy `obj` into a
|
|
282
|
+
| `put(obj, block=True, timeout=None)` | Copy `obj` into a shared-memory block and send it. |
|
|
270
283
|
| `put_nowait(obj)` | `put(obj, block=False)`. |
|
|
271
|
-
| `wait_for_all_ack(timeout=None) -> bool` | Wait until every message is received and
|
|
272
|
-
| `close()` | Free unread messages silently and release resources. Runs automatically at normal process exit. |
|
|
284
|
+
| `wait_for_all_ack(timeout=None) -> bool` | Wait until every message is received and its block released; `False` on timeout. |
|
|
285
|
+
| `close()` | Free unread messages and reusable blocks silently and release resources. Runs automatically at normal process exit. |
|
|
273
286
|
| `closed` | Whether the sender is closed. |
|
|
274
287
|
| `with sender:` | Closes the sender on exit; warns if a clean exit discards unacknowledged messages. |
|
|
275
288
|
|
|
@@ -308,8 +321,9 @@ the last handle closes, so nothing can leak after a crash.
|
|
|
308
321
|
producer closed or exited before this message was read. Call `wait_for_all_ack()` before
|
|
309
322
|
closing the sender; the sender also warns when this happens.
|
|
310
323
|
|
|
311
|
-
**`OSError: [Errno 24] Too many open files` with many unread messages:** fixed in 4.1.1
|
|
312
|
-
|
|
324
|
+
**`OSError: [Errno 24] Too many open files` with many unread messages:** fixed in 4.1.1.
|
|
325
|
+
On Linux and macOS the sender keeps at most 8 blocks mapped for reuse, two file
|
|
326
|
+
descriptors each; other unread messages hold none.
|
|
313
327
|
|
|
314
328
|
**`RuntimeError: A shared-memory pair supports one producing process`:** a second
|
|
315
329
|
process called `put`. Create a separate pair for each producer process.
|
|
@@ -20,8 +20,8 @@ images, and simulation states arrive up to 2× faster, and transfers too large f
|
|
|
20
20
|
`queue.Full`, and `queue.Empty`.
|
|
21
21
|
- **Any picklable object:** NumPy arrays and other buffers use pickle protocol 5
|
|
22
22
|
out-of-band buffers, so they are copied straight into shared memory.
|
|
23
|
-
- **Bounded memory:** a capacity limit applies backpressure, and
|
|
24
|
-
|
|
23
|
+
- **Bounded memory:** a capacity limit applies backpressure, and received blocks are
|
|
24
|
+
reused for later messages, so shared memory stays within `capacity × message size`.
|
|
25
25
|
- **One producer, many consumers:** consumer processes and threads share one receiver.
|
|
26
26
|
- **Portable:** Linux, macOS, and Windows; `spawn`, `fork`, and `forkserver`; Python 3.11–3.15.
|
|
27
27
|
- **Lightweight:** no dependencies, fully typed (PEP 561), and 100% test coverage.
|
|
@@ -81,8 +81,8 @@ arrays, image or video frames, RL rollouts, large `bytes`, or big nested objects
|
|
|
81
81
|
helps where large `multiprocessing.Queue` transfers cause heavy pipe I/O or uneven
|
|
82
82
|
throughput, which is especially noticeable on Windows.
|
|
83
83
|
|
|
84
|
-
Prefer `multiprocessing.Queue` for small messages. Below about 100 kB,
|
|
85
|
-
|
|
84
|
+
Prefer `multiprocessing.Queue` for small messages. Below about 100 kB, the fixed cost of
|
|
85
|
+
mapping a shared-memory block for every message outweighs the pipe.
|
|
86
86
|
|
|
87
87
|
To let several processes read and write **the same** array in place, use
|
|
88
88
|
[`multiprocessing.shared_memory`](https://docs.python.org/3/library/multiprocessing.shared_memory.html)
|
|
@@ -105,20 +105,24 @@ Core i9-14900K, 191.8 GiB RAM, Windows 11, Python 3.12.10, NumPy 2.5.3, measured
|
|
|
105
105
|
2026-10-08. Contiguous float32 arrays, `spawn`, capacity 2, median of 5 runs excluding
|
|
106
106
|
startup and warmup; up to 1,000 messages or 512 MiB per run, at least one message.
|
|
107
107
|
1 GB = 10⁹ bytes. See the [raw results](https://github.com/tobiaswuerth/python_shared_memory_queue/blob/main/benchmarks/results/windows-numpy.json).
|
|
108
|
-
These numbers show transfer speed; I/O load and stability depend on the workload.
|
|
108
|
+
These numbers show transfer speed; I/O load and stability depend on the workload. They
|
|
109
|
+
predate version 4.2, which reuses received blocks and so no longer pays to fault in fresh
|
|
110
|
+
pages on every `put`.
|
|
109
111
|
|
|
110
112
|
## How it works
|
|
111
113
|
|
|
112
114
|
1. `put()` pickles the object with protocol 5. Large buffers, such as NumPy array data,
|
|
113
|
-
stay out-of-band, and everything is copied into one
|
|
114
|
-
|
|
115
|
+
stay out-of-band, and everything is copied into one shared-memory block: a block from
|
|
116
|
+
an earlier, received message when one fits, otherwise a new one.
|
|
117
|
+
2. Only the block's name, buffer sizes, and a sequence number travel through a
|
|
118
|
+
`multiprocessing.Queue`.
|
|
115
119
|
3. `get()` attaches to the block, copies the data into local memory, unpickles it, and
|
|
116
120
|
sends an acknowledgement back.
|
|
117
|
-
4. A background thread in the producer
|
|
118
|
-
arrives. `capacity` limits how many
|
|
121
|
+
4. A background thread in the producer makes the block available for reuse when the
|
|
122
|
+
acknowledgement arrives. `capacity` limits how many messages can be outstanding.
|
|
119
123
|
|
|
120
124
|
Received objects never point into shared memory, so they stay valid after the block is
|
|
121
|
-
freed and can be modified freely.
|
|
125
|
+
reused or freed and can be modified freely.
|
|
122
126
|
|
|
123
127
|
## Usage guide
|
|
124
128
|
|
|
@@ -138,9 +142,15 @@ consumer = ctx.Process(target=consume, args=(receiver,))
|
|
|
138
142
|
|
|
139
143
|
### Capacity and backpressure
|
|
140
144
|
|
|
141
|
-
`capacity` counts messages that were sent but not yet received
|
|
142
|
-
|
|
143
|
-
|
|
145
|
+
`capacity` counts messages that were sent but not yet received. When it is reached,
|
|
146
|
+
`put()` blocks, so a fast producer cannot exhaust memory. Shared-memory use is at most
|
|
147
|
+
about `capacity × message size`. `capacity=0` or `None` means unbounded.
|
|
148
|
+
|
|
149
|
+
Received blocks are reused rather than freed, because writing into memory that is
|
|
150
|
+
already mapped is much faster than faulting in fresh pages. A block is reused for a
|
|
151
|
+
message that fills at least half of it, or for any message that fits a block of up to
|
|
152
|
+
1 MiB. The sender keeps at most 8 such blocks and 256 MiB, never more blocks than the
|
|
153
|
+
capacity allows, and frees them on `close()`. Blocks over 256 MiB are freed once received.
|
|
144
154
|
|
|
145
155
|
### Timeouts and non-blocking calls
|
|
146
156
|
|
|
@@ -191,7 +201,9 @@ such as `None` is a simple way to tell consumers that the stream has ended.
|
|
|
191
201
|
Arrays keep their dtype, shape, and writability. Read-only arrays stay read-only, and
|
|
192
202
|
writable ones arrive as writable local copies. C- and Fortran-contiguous arrays keep
|
|
193
203
|
their layout and are copied straight into shared memory without an intermediate pickle
|
|
194
|
-
copy; other views, such as strided slices, arrive as contiguous copies.
|
|
204
|
+
copy; other views, such as strided slices, arrive as contiguous copies. Each received
|
|
205
|
+
array has its own aligned memory, so keeping one array does not keep the rest of the
|
|
206
|
+
message alive.
|
|
195
207
|
|
|
196
208
|
### Errors
|
|
197
209
|
|
|
@@ -200,8 +212,9 @@ copy; other views, such as strided slices, arrive as contiguous copies.
|
|
|
200
212
|
| `queue.Full` | `put` found no capacity within `timeout` or with `block=False`. |
|
|
201
213
|
| `queue.Empty` | `get` found no message within `timeout` or with `block=False`. |
|
|
202
214
|
| `BrokenPipeError` | The endpoint is closed, or the producer's acknowledgement channel failed. |
|
|
215
|
+
| `FileNotFoundError` | `get` received a message that the producer freed by closing before it was read. |
|
|
203
216
|
| `RuntimeError` | A second process tries to produce, or a used sender is passed to another process. |
|
|
204
|
-
| Pickling errors | The object cannot be pickled or unpickled. Both endpoints stay usable, and failed messages are still
|
|
217
|
+
| Pickling errors | The object cannot be pickled or unpickled. Both endpoints stay usable, and failed messages are still released. |
|
|
205
218
|
| `RuntimeWarning` (warning) | A `with sender:` block or the producing process ended normally and discarded unacknowledged messages. |
|
|
206
219
|
|
|
207
220
|
## API reference
|
|
@@ -216,10 +229,10 @@ for the underlying queues; it defaults to `multiprocessing.get_context()`.
|
|
|
216
229
|
|
|
217
230
|
| Member | Description |
|
|
218
231
|
| --- | --- |
|
|
219
|
-
| `put(obj, block=True, timeout=None)` | Copy `obj` into a
|
|
232
|
+
| `put(obj, block=True, timeout=None)` | Copy `obj` into a shared-memory block and send it. |
|
|
220
233
|
| `put_nowait(obj)` | `put(obj, block=False)`. |
|
|
221
|
-
| `wait_for_all_ack(timeout=None) -> bool` | Wait until every message is received and
|
|
222
|
-
| `close()` | Free unread messages silently and release resources. Runs automatically at normal process exit. |
|
|
234
|
+
| `wait_for_all_ack(timeout=None) -> bool` | Wait until every message is received and its block released; `False` on timeout. |
|
|
235
|
+
| `close()` | Free unread messages and reusable blocks silently and release resources. Runs automatically at normal process exit. |
|
|
223
236
|
| `closed` | Whether the sender is closed. |
|
|
224
237
|
| `with sender:` | Closes the sender on exit; warns if a clean exit discards unacknowledged messages. |
|
|
225
238
|
|
|
@@ -258,8 +271,9 @@ the last handle closes, so nothing can leak after a crash.
|
|
|
258
271
|
producer closed or exited before this message was read. Call `wait_for_all_ack()` before
|
|
259
272
|
closing the sender; the sender also warns when this happens.
|
|
260
273
|
|
|
261
|
-
**`OSError: [Errno 24] Too many open files` with many unread messages:** fixed in 4.1.1
|
|
262
|
-
|
|
274
|
+
**`OSError: [Errno 24] Too many open files` with many unread messages:** fixed in 4.1.1.
|
|
275
|
+
On Linux and macOS the sender keeps at most 8 blocks mapped for reuse, two file
|
|
276
|
+
descriptors each; other unread messages hold none.
|
|
263
277
|
|
|
264
278
|
**`RuntimeError: A shared-memory pair supports one producing process`:** a second
|
|
265
279
|
process called `put`. Create a separate pair for each producer process.
|
|
@@ -25,5 +25,5 @@ from .factory import create_shared_memory_pair
|
|
|
25
25
|
from .receiver import SharedMemoryReceiver
|
|
26
26
|
from .sender import SharedMemorySender
|
|
27
27
|
|
|
28
|
-
__version__ = "4.
|
|
28
|
+
__version__ = "4.2.0"
|
|
29
29
|
__all__ = ["SharedMemoryReceiver", "SharedMemorySender", "__version__", "create_shared_memory_pair"]
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import errno
|
|
4
4
|
import pickle
|
|
5
5
|
import sys
|
|
6
|
+
from collections.abc import Callable
|
|
6
7
|
from contextlib import suppress
|
|
7
8
|
from multiprocessing.shared_memory import SharedMemory
|
|
8
9
|
from typing import Any, NamedTuple, cast
|
|
@@ -15,12 +16,18 @@ _ATTACH_OPTIONS: dict[str, Any] = {"track": False} if sys.version_info >= (3, 13
|
|
|
15
16
|
class SharedMemoryMessage(NamedTuple):
|
|
16
17
|
name: str
|
|
17
18
|
buffer_sizes: tuple[int, ...]
|
|
19
|
+
# Unique per sender. Blocks are reused, so acknowledgements name the message, not the block.
|
|
20
|
+
sequence: int = 0
|
|
18
21
|
|
|
19
22
|
@property
|
|
20
23
|
def size(self) -> int:
|
|
21
24
|
return sum(self.buffer_sizes)
|
|
22
25
|
|
|
23
26
|
|
|
27
|
+
def create_shared_memory(size: int) -> SharedMemory:
|
|
28
|
+
return SharedMemory(create=True, size=size)
|
|
29
|
+
|
|
30
|
+
|
|
24
31
|
def release_shared_memory(allocation: SharedMemory) -> None:
|
|
25
32
|
"""Release both the mapping and its name, including on error paths."""
|
|
26
33
|
try:
|
|
@@ -30,8 +37,15 @@ def release_shared_memory(allocation: SharedMemory) -> None:
|
|
|
30
37
|
allocation.unlink()
|
|
31
38
|
|
|
32
39
|
|
|
33
|
-
def serialize(
|
|
34
|
-
|
|
40
|
+
def serialize(
|
|
41
|
+
data: Any,
|
|
42
|
+
sequence: int = 0,
|
|
43
|
+
allocate: Callable[[int], SharedMemory] = create_shared_memory,
|
|
44
|
+
) -> tuple[SharedMemory, SharedMemoryMessage]:
|
|
45
|
+
"""Copy a pickle and its out-of-band buffers into an owned block of at least that size.
|
|
46
|
+
|
|
47
|
+
`allocate` returns a new or reused block; this function frees it if copying fails.
|
|
48
|
+
"""
|
|
35
49
|
buffers: list[memoryview] = []
|
|
36
50
|
allocation = None
|
|
37
51
|
|
|
@@ -41,14 +55,14 @@ def serialize(data: Any) -> tuple[SharedMemory, SharedMemoryMessage]:
|
|
|
41
55
|
try:
|
|
42
56
|
pickle_data = pickle.dumps(data, protocol=5, buffer_callback=append_buffer)
|
|
43
57
|
buffer_sizes = (len(pickle_data), *(buffer.nbytes for buffer in buffers))
|
|
44
|
-
allocation =
|
|
58
|
+
allocation = allocate(sum(buffer_sizes))
|
|
45
59
|
shared = cast(memoryview, allocation.buf)
|
|
46
60
|
shared[: len(pickle_data)] = pickle_data
|
|
47
61
|
offset = len(pickle_data)
|
|
48
62
|
for buffer in buffers:
|
|
49
63
|
shared[offset : offset + buffer.nbytes] = buffer
|
|
50
64
|
offset += buffer.nbytes
|
|
51
|
-
return allocation, SharedMemoryMessage(allocation.name, buffer_sizes)
|
|
65
|
+
return allocation, SharedMemoryMessage(allocation.name, buffer_sizes, sequence)
|
|
52
66
|
except BaseException:
|
|
53
67
|
if allocation is not None:
|
|
54
68
|
release_shared_memory(allocation)
|
|
@@ -60,25 +74,41 @@ def serialize(data: Any) -> tuple[SharedMemory, SharedMemoryMessage]:
|
|
|
60
74
|
|
|
61
75
|
def deserialize(message: SharedMemoryMessage) -> Any:
|
|
62
76
|
"""Unpickle with owned local buffers and always close the shared mapping."""
|
|
77
|
+
return load(message, attach(message))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def attach(message: SharedMemoryMessage) -> SharedMemory:
|
|
81
|
+
"""Map the block that valid metadata names, or explain why it no longer exists."""
|
|
63
82
|
_validate_message(message)
|
|
64
83
|
try:
|
|
65
84
|
allocation = SharedMemory(name=message.name, **_ATTACH_OPTIONS)
|
|
66
85
|
except FileNotFoundError as error:
|
|
67
|
-
raise
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
)
|
|
86
|
+
raise _freed_block_error(message) from error
|
|
87
|
+
# Attaching on Windows is not atomic: a name freed in between is recreated as a new,
|
|
88
|
+
# zero-filled block. Every pickle starts with a nonzero opcode.
|
|
89
|
+
if cast(memoryview, allocation.buf)[0] == 0:
|
|
90
|
+
allocation.close()
|
|
91
|
+
raise _freed_block_error(message)
|
|
92
|
+
return allocation
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _freed_block_error(message: SharedMemoryMessage) -> FileNotFoundError:
|
|
96
|
+
return FileNotFoundError(
|
|
97
|
+
errno.ENOENT,
|
|
98
|
+
"Shared-memory block was freed before it was received; the sender probably "
|
|
99
|
+
"closed without calling wait_for_all_ack()",
|
|
100
|
+
message.name,
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def load(message: SharedMemoryMessage, allocation: SharedMemory) -> Any:
|
|
105
|
+
"""Unpickle an attached block with owned local buffers, then close the mapping."""
|
|
73
106
|
try:
|
|
74
107
|
if message.size > allocation.size:
|
|
75
108
|
raise ValueError("Shared-memory block is smaller than its metadata")
|
|
76
109
|
shared = cast(memoryview, allocation.buf)
|
|
77
110
|
pickle_size = message.buffer_sizes[0]
|
|
78
|
-
buffers
|
|
79
|
-
if len(message.buffer_sizes) > 1:
|
|
80
|
-
with shared[pickle_size : message.size] as shared_buffer:
|
|
81
|
-
buffers = _copy_buffers(shared_buffer, message.buffer_sizes[1:])
|
|
111
|
+
buffers = _copy_buffers(shared, pickle_size, message.buffer_sizes[1:])
|
|
82
112
|
with shared[:pickle_size] as pickle_data:
|
|
83
113
|
return pickle.loads(pickle_data, buffers=buffers)
|
|
84
114
|
finally:
|
|
@@ -91,6 +121,8 @@ def _validate_message(message: SharedMemoryMessage) -> None:
|
|
|
91
121
|
sizes = message.buffer_sizes
|
|
92
122
|
if (
|
|
93
123
|
not message.name
|
|
124
|
+
or type(message.sequence) is not int
|
|
125
|
+
or message.sequence < 0
|
|
94
126
|
or not isinstance(sizes, tuple)
|
|
95
127
|
or not sizes
|
|
96
128
|
or any(type(size) is not int or size < 0 for size in sizes)
|
|
@@ -99,12 +131,12 @@ def _validate_message(message: SharedMemoryMessage) -> None:
|
|
|
99
131
|
raise ValueError("Invalid shared-memory metadata")
|
|
100
132
|
|
|
101
133
|
|
|
102
|
-
def _copy_buffers(
|
|
103
|
-
#
|
|
134
|
+
def _copy_buffers(shared: memoryview, offset: int, sizes: tuple[int, ...]) -> list[memoryview]:
|
|
135
|
+
# A separate writable copy per buffer starts aligned for any dtype and is freed on its own,
|
|
136
|
+
# so a small array cannot keep a large neighbour alive.
|
|
104
137
|
buffers = []
|
|
105
|
-
|
|
106
|
-
offset
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
offset += size
|
|
138
|
+
for size in sizes:
|
|
139
|
+
with shared[offset : offset + size] as shared_buffer:
|
|
140
|
+
buffers.append(memoryview(bytearray(shared_buffer)))
|
|
141
|
+
offset += size
|
|
110
142
|
return buffers
|
|
@@ -6,18 +6,18 @@ from multiprocessing.queues import Queue
|
|
|
6
6
|
from types import TracebackType
|
|
7
7
|
from typing import Any, Self
|
|
8
8
|
|
|
9
|
-
from .convert import SharedMemoryMessage,
|
|
9
|
+
from .convert import SharedMemoryMessage, attach, load
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class SharedMemoryReceiver:
|
|
13
13
|
"""Consuming endpoint of a pair; create it with `create_shared_memory_pair`.
|
|
14
14
|
|
|
15
15
|
Several threads and processes may share one receiver; each message is delivered to
|
|
16
|
-
exactly one of them. Every
|
|
17
|
-
the sender can free its memory.
|
|
16
|
+
exactly one of them. Every received block is acknowledged, even when decoding fails,
|
|
17
|
+
so the sender can free its memory.
|
|
18
18
|
"""
|
|
19
19
|
|
|
20
|
-
def __init__(self, data_queue: Queue[SharedMemoryMessage], ack_queue: Queue[
|
|
20
|
+
def __init__(self, data_queue: Queue[SharedMemoryMessage], ack_queue: Queue[int]) -> None:
|
|
21
21
|
self._data_queue = data_queue
|
|
22
22
|
self._ack_queue = ack_queue
|
|
23
23
|
self._closed = False
|
|
@@ -35,24 +35,26 @@ class SharedMemoryReceiver:
|
|
|
35
35
|
timeout: Maximum seconds to wait; `None` waits forever.
|
|
36
36
|
|
|
37
37
|
Returns:
|
|
38
|
-
The object, which stays valid after the
|
|
38
|
+
The object, which stays valid after the block is reused or freed. Arrays keep
|
|
39
39
|
their original writability.
|
|
40
40
|
|
|
41
41
|
Raises:
|
|
42
42
|
queue.Empty: No message arrived in time.
|
|
43
43
|
BrokenPipeError: The receiver is closed.
|
|
44
|
+
FileNotFoundError: The sender freed the message before it was received.
|
|
44
45
|
|
|
45
46
|
Unpickling errors propagate unchanged; the message is still acknowledged.
|
|
46
47
|
"""
|
|
47
48
|
if self._closed:
|
|
48
49
|
raise BrokenPipeError("Receiver is closed")
|
|
49
50
|
message = self._data_queue.get(block, timeout)
|
|
51
|
+
# A block that is gone was already freed by its sender, which expects no acknowledgement.
|
|
52
|
+
allocation = attach(message)
|
|
50
53
|
try:
|
|
51
|
-
return
|
|
54
|
+
return load(message, allocation)
|
|
52
55
|
finally:
|
|
53
56
|
# Failed decoding must still release the sender's memory and capacity.
|
|
54
|
-
|
|
55
|
-
self._ack_queue.put_nowait(message.name)
|
|
57
|
+
self._ack_queue.put_nowait(message.sequence)
|
|
56
58
|
|
|
57
59
|
def get_nowait(self) -> Any:
|
|
58
60
|
"""Equivalent to `get(block=False)`."""
|