zprocess 2.27.1__tar.gz → 2.28.0rc2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/.github/workflows/release.yml +17 -11
- {zprocess-2.27.1/zprocess.egg-info → zprocess-2.28.0rc2}/PKG-INFO +9 -3
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/pyproject.toml +10 -4
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/requirements.txt +4 -1
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/tests/test_zprocess.py +280 -82
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/__version__.py +1 -1
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/process_tree.py +48 -20
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/utils.py +8 -1
- {zprocess-2.27.1 → zprocess-2.28.0rc2/zprocess.egg-info}/PKG-INFO +9 -3
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess.egg-info/SOURCES.txt +2 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess.egg-info/requires.txt +4 -1
- zprocess-2.28.0rc2/zprocess.egg-info/scm_file_list.json +51 -0
- zprocess-2.28.0rc2/zprocess.egg-info/scm_version.json +8 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/.github/workflows/release-vars.sh +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/.gitignore +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/LICENSE.txt +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/README.md +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/doc/Makefile +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/doc/conf.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/doc/index.rst +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/doc/requirements.txt +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/pytest.ini +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/run_tests.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/setup.cfg +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/tests/GUI_output_redirection.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/tests/output_redirection.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/tests/test_remote.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/tests/test_zlock_server.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/tests/test_zlog_server.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/__init__.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/clientserver.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/cryptoproxy.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/examples/example_child.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/examples/example_parent.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/examples/processclass_example.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/examples/processclass_remote_example.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/locking/__init__.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/locking/__main__.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/makesecret.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/process_class_wrapper.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/remote/__init__.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/remote/__main__.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/remote/curses_server.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/remote/server.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/security.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/socks/__init__.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/socks/server.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/tasks.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/zlock/__init__.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/zlock/__main__.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/zlock/server.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/zlog/__init__.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/zlog/__main__.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess/zlog/server.py +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess.egg-info/dependency_links.txt +0 -0
- {zprocess-2.27.1 → zprocess-2.28.0rc2}/zprocess.egg-info/top_level.txt +0 -0
|
@@ -10,7 +10,7 @@ on:
|
|
|
10
10
|
|
|
11
11
|
env:
|
|
12
12
|
OS_LIST_UBUNTU: '["ubuntu-latest"]'
|
|
13
|
-
OS_LIST_ALL: '["ubuntu-latest", "windows-
|
|
13
|
+
OS_LIST_ALL: '["ubuntu-latest", "windows-2022", "macos-latest", "macos-15-intel"]'
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
jobs:
|
|
@@ -38,9 +38,12 @@ jobs:
|
|
|
38
38
|
with:
|
|
39
39
|
fetch-depth: 0
|
|
40
40
|
|
|
41
|
-
- name: Ignore
|
|
42
|
-
|
|
43
|
-
|
|
41
|
+
- name: Ignore tags other than the pushed tag
|
|
42
|
+
# The version comes from the tags at HEAD, so a commit that carries both an rc
|
|
43
|
+
# tag and a final tag builds the version of the tag that was pushed, and a branch
|
|
44
|
+
# push of a tagged commit builds a development version.
|
|
45
|
+
shell: bash
|
|
46
|
+
run: git tag -d $(git tag --points-at HEAD | grep -vx "${GITHUB_REF_NAME}" || true)
|
|
44
47
|
|
|
45
48
|
- name: Install Python
|
|
46
49
|
uses: actions/setup-python@v6
|
|
@@ -226,9 +229,12 @@ jobs:
|
|
|
226
229
|
with:
|
|
227
230
|
fetch-depth: 0
|
|
228
231
|
|
|
229
|
-
- name: Ignore
|
|
230
|
-
|
|
231
|
-
|
|
232
|
+
- name: Ignore tags other than the pushed tag
|
|
233
|
+
# The version comes from the tags at HEAD, so a commit that carries both an rc
|
|
234
|
+
# tag and a final tag builds the version of the tag that was pushed, and a branch
|
|
235
|
+
# push of a tagged commit builds a development version.
|
|
236
|
+
shell: bash
|
|
237
|
+
run: git tag -d $(git tag --points-at HEAD | grep -vx "${GITHUB_REF_NAME}" || true)
|
|
232
238
|
|
|
233
239
|
- name: Install Python
|
|
234
240
|
uses: actions/setup-python@v6
|
|
@@ -259,7 +265,7 @@ jobs:
|
|
|
259
265
|
if-no-files-found: error
|
|
260
266
|
|
|
261
267
|
- name: Install Miniforge
|
|
262
|
-
uses: conda-incubator/setup-miniconda@
|
|
268
|
+
uses: conda-incubator/setup-miniconda@v4
|
|
263
269
|
with:
|
|
264
270
|
miniforge-version: "latest"
|
|
265
271
|
auto-update-conda: true
|
|
@@ -305,7 +311,7 @@ jobs:
|
|
|
305
311
|
merge-multiple: true
|
|
306
312
|
|
|
307
313
|
- name: Create GitHub release and upload release asset
|
|
308
|
-
uses: softprops/action-gh-release@
|
|
314
|
+
uses: softprops/action-gh-release@v3
|
|
309
315
|
env:
|
|
310
316
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
311
317
|
with:
|
|
@@ -390,7 +396,7 @@ jobs:
|
|
|
390
396
|
merge-multiple: true
|
|
391
397
|
|
|
392
398
|
- name: Install Miniforge
|
|
393
|
-
uses: conda-incubator/setup-miniconda@
|
|
399
|
+
uses: conda-incubator/setup-miniconda@v4
|
|
394
400
|
with:
|
|
395
401
|
miniforge-version: "latest"
|
|
396
402
|
auto-update-conda: true
|
|
@@ -429,7 +435,7 @@ jobs:
|
|
|
429
435
|
merge-multiple: true
|
|
430
436
|
|
|
431
437
|
- name: Install Miniforge
|
|
432
|
-
uses: conda-incubator/setup-miniconda@
|
|
438
|
+
uses: conda-incubator/setup-miniconda@v4
|
|
433
439
|
with:
|
|
434
440
|
miniforge-version: "latest"
|
|
435
441
|
auto-update-conda: true
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: zprocess
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.28.0rc2
|
|
4
4
|
Summary: A set of utilities for multiprocessing using zeromq
|
|
5
5
|
Author-email: Christopher Billington <chrisjbillington@gmail.com>
|
|
6
6
|
License-Expression: BSD-2-Clause
|
|
@@ -11,11 +11,17 @@ Project-URL: Downloads, https://pypi.org/project/zprocess/
|
|
|
11
11
|
Project-URL: Tracker, https://github.com/chrisjbillington/zprocess/issues
|
|
12
12
|
Keywords: multiprocessing,IPC,zeromq
|
|
13
13
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
-
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
19
|
+
Requires-Python: >=3.11
|
|
15
20
|
Description-Content-Type: text/markdown
|
|
16
21
|
License-File: LICENSE.txt
|
|
17
|
-
Requires-Dist: setuptools_scm
|
|
22
|
+
Requires-Dist: setuptools_scm>=10
|
|
18
23
|
Requires-Dist: pyzmq
|
|
24
|
+
Requires-Dist: appnope; sys_platform == "darwin"
|
|
19
25
|
Requires-Dist: pywin32; sys_platform == "win32"
|
|
20
26
|
Requires-Dist: windows-curses; sys_platform == "win32"
|
|
21
27
|
Provides-Extra: test
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["setuptools", "setuptools_scm"]
|
|
2
|
+
requires = ["setuptools", "setuptools_scm>=10"]
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
5
|
[tool.setuptools_scm]
|
|
6
|
-
version_scheme = "release-branch
|
|
6
|
+
version_scheme = "semver-pep440-release-branch"
|
|
7
7
|
local_scheme = "no-local-version"
|
|
8
8
|
|
|
9
9
|
[tool.setuptools.packages]
|
|
@@ -20,11 +20,17 @@ license = "BSD-2-Clause"
|
|
|
20
20
|
license-files = ["LICENSE.txt"]
|
|
21
21
|
classifiers = [
|
|
22
22
|
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Programming Language :: Python :: 3.14",
|
|
27
|
+
"Programming Language :: Python :: 3.15",
|
|
23
28
|
]
|
|
24
|
-
requires-python = ">=3.
|
|
29
|
+
requires-python = ">=3.11"
|
|
25
30
|
dependencies = [
|
|
26
|
-
"setuptools_scm",
|
|
31
|
+
"setuptools_scm>=10",
|
|
27
32
|
"pyzmq",
|
|
33
|
+
"appnope; sys_platform == 'darwin'",
|
|
28
34
|
"pywin32; sys_platform == 'win32'",
|
|
29
35
|
"windows-curses; sys_platform == 'win32'",
|
|
30
36
|
]
|
|
@@ -235,9 +235,9 @@ class HeartbeatClientTestProcess(Process):
|
|
|
235
235
|
time.sleep(10)
|
|
236
236
|
|
|
237
237
|
|
|
238
|
-
class HeartbeatServerTestProcess(Process):
|
|
239
|
-
"""For testing that parent processes are behaving correcly re. heartbeats"""
|
|
240
|
-
def run(self):
|
|
238
|
+
class HeartbeatServerTestProcess(Process):
|
|
239
|
+
"""For testing that parent processes are behaving correcly re. heartbeats"""
|
|
240
|
+
def run(self):
|
|
241
241
|
# We'll send heartbeats of our own, independent of the HeartbeatClient
|
|
242
242
|
# already running in this process:
|
|
243
243
|
shared_secret = self.process_tree.shared_secret
|
|
@@ -256,89 +256,287 @@ class HeartbeatServerTestProcess(Process):
|
|
|
256
256
|
self.to_parent.put(True)
|
|
257
257
|
time.sleep(1) # Ensure it sends before we return
|
|
258
258
|
return
|
|
259
|
-
self.to_parent.put(False)
|
|
260
|
-
time.sleep(1) # Ensure it sends before we return
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
class
|
|
264
|
-
def
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
self.
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
self.
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
self.
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
self.
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
self.
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
259
|
+
self.to_parent.put(False)
|
|
260
|
+
time.sleep(1) # Ensure it sends before we return
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
class ScriptedHeartbeatServer(object):
|
|
264
|
+
def __init__(self, shared_secret, script, default='good'):
|
|
265
|
+
import zmq
|
|
266
|
+
|
|
267
|
+
self.context = SecureContext.instance(shared_secret=shared_secret)
|
|
268
|
+
self.script = list(script)
|
|
269
|
+
self.default = default
|
|
270
|
+
self.actions = []
|
|
271
|
+
self.request_times = []
|
|
272
|
+
self._lock = threading.Lock()
|
|
273
|
+
self._running = True
|
|
274
|
+
self._request_event = threading.Event()
|
|
275
|
+
self.sock = self.context.socket(zmq.ROUTER)
|
|
276
|
+
self.port = self.sock.bind_to_random_port('tcp://127.0.0.1')
|
|
277
|
+
self.thread = threading.Thread(target=self.mainloop)
|
|
278
|
+
self.thread.daemon = True
|
|
279
|
+
self.thread.start()
|
|
280
|
+
|
|
281
|
+
def _next_action(self):
|
|
282
|
+
with self._lock:
|
|
283
|
+
if self.script:
|
|
284
|
+
action = self.script.pop(0)
|
|
285
|
+
else:
|
|
286
|
+
action = self.default
|
|
287
|
+
self.actions.append(action)
|
|
288
|
+
return action
|
|
289
|
+
|
|
290
|
+
def request_count(self):
|
|
291
|
+
with self._lock:
|
|
292
|
+
return len(self.request_times)
|
|
293
|
+
|
|
294
|
+
def wait_for_requests(self, count, timeout):
|
|
295
|
+
deadline = time.time() + timeout
|
|
296
|
+
while time.time() < deadline:
|
|
297
|
+
if self.request_count() >= count:
|
|
298
|
+
return True
|
|
299
|
+
self._request_event.wait(min(0.1, deadline - time.time()))
|
|
300
|
+
self._request_event.clear()
|
|
301
|
+
return self.request_count() >= count
|
|
302
|
+
|
|
303
|
+
def inter_request_intervals(self):
|
|
304
|
+
with self._lock:
|
|
305
|
+
times = list(self.request_times)
|
|
306
|
+
return [later - earlier for earlier, later in zip(times, times[1:])]
|
|
307
|
+
|
|
308
|
+
def mainloop(self):
|
|
309
|
+
while self._running:
|
|
310
|
+
try:
|
|
311
|
+
if not self.sock.poll(100):
|
|
312
|
+
continue
|
|
313
|
+
msg = self.sock.recv_multipart()
|
|
314
|
+
with self._lock:
|
|
315
|
+
self.request_times.append(time.time())
|
|
316
|
+
self._request_event.set()
|
|
317
|
+
action = self._next_action()
|
|
318
|
+
payload = msg[-1]
|
|
319
|
+
if action == 'good':
|
|
320
|
+
self.sock.send_multipart(msg[:-1] + [payload])
|
|
321
|
+
elif action == 'malformed':
|
|
322
|
+
self.sock.send_multipart(msg[:-1] + [payload + b'wrong'])
|
|
323
|
+
elif action == 'drop':
|
|
324
|
+
continue
|
|
325
|
+
else:
|
|
326
|
+
raise ValueError(action)
|
|
327
|
+
except zmq.ZMQError:
|
|
328
|
+
if self._running:
|
|
329
|
+
raise
|
|
330
|
+
|
|
331
|
+
def close(self):
|
|
332
|
+
self._running = False
|
|
333
|
+
self.sock.close(linger=0)
|
|
334
|
+
self.thread.join(timeout=1)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
class HeartbeatTests(unittest.TestCase):
|
|
338
|
+
def setUp(self):
|
|
339
|
+
self.heartbeat_server = None
|
|
340
|
+
|
|
341
|
+
def _process_tree(
|
|
342
|
+
self, allowed_missed_heartbeats=None, heartbeat_interval=None
|
|
343
|
+
):
|
|
344
|
+
if allowed_missed_heartbeats is None and heartbeat_interval is None:
|
|
345
|
+
return _default_process_tree
|
|
346
|
+
|
|
347
|
+
class ConfiguredHeartbeatProcessTree(type(_default_process_tree)):
|
|
348
|
+
def subprocess(tree_self, *args, **kwargs):
|
|
349
|
+
if allowed_missed_heartbeats is not None:
|
|
350
|
+
kwargs.setdefault(
|
|
351
|
+
'allowed_missed_heartbeats', allowed_missed_heartbeats
|
|
352
|
+
)
|
|
353
|
+
if heartbeat_interval is not None:
|
|
354
|
+
kwargs.setdefault('heartbeat_interval', heartbeat_interval)
|
|
355
|
+
return super(
|
|
356
|
+
ConfiguredHeartbeatProcessTree, tree_self
|
|
357
|
+
).subprocess(*args, **kwargs)
|
|
358
|
+
|
|
359
|
+
process_tree = ConfiguredHeartbeatProcessTree(
|
|
360
|
+
shared_secret=shared_secret,
|
|
361
|
+
allow_insecure=_default_process_tree.allow_insecure,
|
|
362
|
+
zlock_host=_default_process_tree.zlock_host,
|
|
363
|
+
zlock_port=_default_process_tree.zlock_port,
|
|
364
|
+
zlog_host=_default_process_tree.zlog_host,
|
|
365
|
+
zlog_port=_default_process_tree.zlog_port,
|
|
366
|
+
)
|
|
367
|
+
process_tree.heartbeat_server = self.heartbeat_server
|
|
368
|
+
return process_tree
|
|
369
|
+
|
|
370
|
+
def _start_heartbeat_process(
|
|
371
|
+
self,
|
|
372
|
+
allowed_missed_heartbeats=None,
|
|
373
|
+
heartbeat_interval=None,
|
|
374
|
+
script=None,
|
|
375
|
+
default='good',
|
|
376
|
+
):
|
|
377
|
+
self.heartbeat_server = ScriptedHeartbeatServer(
|
|
378
|
+
shared_secret, script or [], default=default
|
|
379
|
+
)
|
|
380
|
+
process_tree = self._process_tree(
|
|
381
|
+
allowed_missed_heartbeats,
|
|
382
|
+
heartbeat_interval=heartbeat_interval,
|
|
383
|
+
)
|
|
384
|
+
self.process = HeartbeatClientTestProcess(process_tree=process_tree)
|
|
385
|
+
return self.process.start()
|
|
386
|
+
|
|
387
|
+
def _assert_process_exits_within(self, timeout):
|
|
388
|
+
deadline = time.time() + timeout
|
|
389
|
+
while time.time() < deadline:
|
|
390
|
+
if self.process.child.poll() is not None:
|
|
391
|
+
return
|
|
392
|
+
time.sleep(0.1)
|
|
393
|
+
self.fail('process did not exit within %.1f seconds' % timeout)
|
|
394
|
+
|
|
395
|
+
def _assert_alive_after_requests(self, request_count, timeout):
|
|
396
|
+
self.assertTrue(
|
|
397
|
+
self.heartbeat_server.wait_for_requests(request_count, timeout),
|
|
398
|
+
'server did not receive %d heartbeats' % request_count,
|
|
399
|
+
)
|
|
400
|
+
time.sleep(0.2)
|
|
401
|
+
self.assertIs(self.process.child.poll(), None)
|
|
402
|
+
|
|
403
|
+
def _assert_intervals_are_timeouts(self, minimum_interval):
|
|
404
|
+
intervals = self.heartbeat_server.inter_request_intervals()
|
|
405
|
+
self.assertTrue(intervals, 'expected at least two heartbeat requests')
|
|
406
|
+
self.assertTrue(
|
|
407
|
+
all(interval >= minimum_interval for interval in intervals),
|
|
408
|
+
'expected timeout-sized gaps, got %r' % intervals,
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
def _assert_intervals_are_immediate_retries(self, maximum_interval):
|
|
412
|
+
intervals = self.heartbeat_server.inter_request_intervals()
|
|
413
|
+
self.assertTrue(intervals, 'expected at least two heartbeat requests')
|
|
414
|
+
self.assertTrue(
|
|
415
|
+
all(interval <= maximum_interval for interval in intervals),
|
|
416
|
+
'expected malformed replies to avoid timeout gaps, got %r' % intervals,
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
def test_subproc_lives_with_working_heartbeats(self):
|
|
420
|
+
self._start_heartbeat_process(allowed_missed_heartbeats=5)
|
|
421
|
+
self._assert_alive_after_requests(3, 5)
|
|
422
|
+
|
|
423
|
+
def test_subproc_survives_missing_heartbeats_below_budget(self):
|
|
424
|
+
for failures in range(1, 5):
|
|
425
|
+
with self.subTest(failures=failures):
|
|
426
|
+
self._start_heartbeat_process(
|
|
427
|
+
allowed_missed_heartbeats=failures + 1,
|
|
428
|
+
script=['drop'] * failures + ['good'],
|
|
429
|
+
)
|
|
430
|
+
self._assert_alive_after_requests(failures + 1, (failures + 1) * 3)
|
|
431
|
+
self._assert_intervals_are_timeouts(1.5)
|
|
432
|
+
self.process.terminate()
|
|
433
|
+
self.heartbeat_server.close()
|
|
434
|
+
self.process = None
|
|
435
|
+
self.heartbeat_server = None
|
|
436
|
+
|
|
437
|
+
def test_subproc_dies_after_1_to_5_missing_heartbeats(self):
|
|
438
|
+
for failures in range(1, 6):
|
|
439
|
+
with self.subTest(failures=failures):
|
|
440
|
+
self._start_heartbeat_process(
|
|
441
|
+
allowed_missed_heartbeats=failures,
|
|
442
|
+
script=['drop'] * failures,
|
|
443
|
+
)
|
|
444
|
+
self.assertTrue(
|
|
445
|
+
self.heartbeat_server.wait_for_requests(failures, failures * 3),
|
|
446
|
+
'server did not receive %d dropped heartbeats' % failures,
|
|
447
|
+
)
|
|
448
|
+
self._assert_process_exits_within(3)
|
|
449
|
+
if failures > 1:
|
|
450
|
+
self._assert_intervals_are_timeouts(1.5)
|
|
451
|
+
self.process = None
|
|
452
|
+
self.heartbeat_server.close()
|
|
453
|
+
self.heartbeat_server = None
|
|
454
|
+
|
|
455
|
+
def test_subproc_survives_malformed_heartbeats_below_budget(self):
|
|
456
|
+
for failures in range(1, 5):
|
|
457
|
+
with self.subTest(failures=failures):
|
|
458
|
+
self._start_heartbeat_process(
|
|
459
|
+
allowed_missed_heartbeats=failures + 1,
|
|
460
|
+
script=['malformed'] * failures + ['good'],
|
|
461
|
+
)
|
|
462
|
+
self._assert_alive_after_requests(failures + 1, (failures + 1) * 2)
|
|
463
|
+
self._assert_intervals_are_immediate_retries(1.5)
|
|
464
|
+
self.process.terminate()
|
|
465
|
+
self.heartbeat_server.close()
|
|
466
|
+
self.process = None
|
|
467
|
+
self.heartbeat_server = None
|
|
468
|
+
|
|
469
|
+
def test_subproc_dies_after_1_to_5_malformed_heartbeats(self):
|
|
470
|
+
for failures in range(1, 6):
|
|
471
|
+
with self.subTest(failures=failures):
|
|
472
|
+
self._start_heartbeat_process(
|
|
473
|
+
allowed_missed_heartbeats=failures,
|
|
474
|
+
script=['malformed'] * failures,
|
|
475
|
+
)
|
|
476
|
+
self.assertTrue(
|
|
477
|
+
self.heartbeat_server.wait_for_requests(failures, failures * 2),
|
|
478
|
+
'server did not receive %d malformed heartbeats' % failures,
|
|
479
|
+
)
|
|
480
|
+
self._assert_process_exits_within(2)
|
|
481
|
+
if failures > 1:
|
|
482
|
+
self._assert_intervals_are_immediate_retries(1.5)
|
|
483
|
+
self.process = None
|
|
484
|
+
self.heartbeat_server.close()
|
|
485
|
+
self.heartbeat_server = None
|
|
486
|
+
|
|
487
|
+
def test_successful_heartbeat_resets_miss_counter(self):
|
|
488
|
+
self._start_heartbeat_process(
|
|
489
|
+
allowed_missed_heartbeats=2,
|
|
490
|
+
script=['malformed', 'good', 'malformed', 'good'],
|
|
491
|
+
)
|
|
492
|
+
self._assert_alive_after_requests(4, 8)
|
|
493
|
+
self._assert_intervals_are_immediate_retries(1.5)
|
|
494
|
+
|
|
495
|
+
def test_configured_heartbeat_interval_controls_request_cadence(self):
|
|
496
|
+
self._start_heartbeat_process(
|
|
497
|
+
allowed_missed_heartbeats=3,
|
|
498
|
+
heartbeat_interval=0.2,
|
|
499
|
+
)
|
|
500
|
+
self.assertTrue(self.heartbeat_server.wait_for_requests(3, 3))
|
|
501
|
+
intervals = self.heartbeat_server.inter_request_intervals()[:2]
|
|
502
|
+
self.assertTrue(
|
|
503
|
+
all(0.1 <= interval < 0.8 for interval in intervals),
|
|
504
|
+
'expected configured heartbeat-sized gaps, got %r' % intervals,
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
def test_subproc_survives_until_kill_lock_released(self):
|
|
508
|
+
to_child, from_child = self._start_heartbeat_process(
|
|
509
|
+
allowed_missed_heartbeats=1, script=['good', 'drop']
|
|
510
|
+
)
|
|
511
|
+
self.assertTrue(self.heartbeat_server.wait_for_requests(1, 3))
|
|
512
|
+
# Tell child to acquire kill lock for 3 sec:
|
|
513
|
+
to_child.put(None)
|
|
514
|
+
# Don't respond to the heartbeat, process should still be alive 2 sec later
|
|
515
|
+
time.sleep(2)
|
|
516
|
+
# Process should be alive:
|
|
517
|
+
self.assertIs(self.process.child.poll(), None)
|
|
518
|
+
# After kill lock released, child should be terminated:
|
|
519
|
+
self._assert_process_exits_within(3)
|
|
520
|
+
# Process should be dead:
|
|
521
|
+
self.assertIsNot(self.process.child.poll(), None)
|
|
522
|
+
|
|
523
|
+
def test_parent_correctly_responds_to_heartbeats(self):
|
|
524
|
+
# No mock server this time, we're testing the real one:
|
|
525
|
+
_default_process_tree.heartbeat_server = None
|
|
526
|
+
self.process = HeartbeatServerTestProcess()
|
|
331
527
|
to_child, from_child = self.process.start()
|
|
332
528
|
to_child.put(_default_process_tree.heartbeat_server.port)
|
|
333
529
|
self.assertTrue(from_child.sock.poll(1000))
|
|
334
530
|
self.assertTrue(from_child.get())
|
|
335
|
-
|
|
336
|
-
def tearDown(self):
|
|
337
|
-
self.
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
531
|
+
|
|
532
|
+
def tearDown(self):
|
|
533
|
+
if self.heartbeat_server is not None:
|
|
534
|
+
self.heartbeat_server.close()
|
|
535
|
+
try:
|
|
536
|
+
if getattr(self, 'process', None) is not None:
|
|
537
|
+
self.process.terminate()
|
|
538
|
+
except Exception:
|
|
539
|
+
pass # already dead
|
|
342
540
|
|
|
343
541
|
class TestEventProcess(Process):
|
|
344
542
|
def run(self):
|
|
@@ -6,7 +6,7 @@ if (root / '.git').is_dir():
|
|
|
6
6
|
from setuptools_scm import get_version
|
|
7
7
|
|
|
8
8
|
__version__ = get_version(
|
|
9
|
-
root, version_scheme="release-branch
|
|
9
|
+
root, version_scheme="semver-pep440-release-branch", local_scheme="no-local-version"
|
|
10
10
|
)
|
|
11
11
|
else:
|
|
12
12
|
try:
|
|
@@ -156,6 +156,11 @@ class HeartbeatServer(object):
|
|
|
156
156
|
context = SecureContext.instance(shared_secret=shared_secret)
|
|
157
157
|
self.sock = context.socket(zmq.REP)
|
|
158
158
|
self.port = self.sock.bind_to_random_port(bind_address)
|
|
159
|
+
if sys.platform == 'darwin':
|
|
160
|
+
# App Nap delays the replies of an idle GUI process, and its subprocesses
|
|
161
|
+
# would take it for dead:
|
|
162
|
+
import appnope
|
|
163
|
+
appnope.nope()
|
|
159
164
|
self.mainloop_thread = threading.Thread(target=self.mainloop)
|
|
160
165
|
self.mainloop_thread.daemon = True
|
|
161
166
|
self.mainloop_thread.start()
|
|
@@ -170,45 +175,56 @@ class HeartbeatServer(object):
|
|
|
170
175
|
|
|
171
176
|
|
|
172
177
|
class HeartbeatClient(object):
|
|
178
|
+
"""A heartbeating thread that terminates the process, unless a lock is held,
|
|
179
|
+
once allowed_missed_heartbeats heartbeats in a row have had no correct reply."""
|
|
180
|
+
|
|
181
|
+
DEFAULT_ALLOWED_MISSED_HEARTBEATS = 5
|
|
173
182
|
|
|
174
183
|
# How long between heartbeats?
|
|
175
184
|
HEARTBEAT_INTERVAL = 1
|
|
176
|
-
# How long
|
|
177
|
-
#
|
|
185
|
+
# How long to wait for each reply? Wait longer when the heartbeat server is
|
|
186
|
+
# remote to be more forgiving of the network:
|
|
178
187
|
LOCALHOST_TIMEOUT = 1
|
|
179
188
|
REMOTE_TIMEOUT = 10
|
|
180
189
|
|
|
181
|
-
"""A heartbeating thread that terminates the process if it doesn't get the
|
|
182
|
-
heartbeats back within one second, unless a lock is held."""
|
|
183
190
|
def __init__(self, server_host, server_port,
|
|
184
|
-
shared_secret=None, allow_insecure=False
|
|
191
|
+
shared_secret=None, allow_insecure=False,
|
|
192
|
+
heartbeat_interval=HEARTBEAT_INTERVAL,
|
|
193
|
+
allowed_missed_heartbeats=DEFAULT_ALLOWED_MISSED_HEARTBEATS):
|
|
185
194
|
self.lock = KillLock()
|
|
186
|
-
context = SecureContext.instance(shared_secret=shared_secret)
|
|
187
|
-
self.sock = context.socket(zmq.REQ, allow_insecure=allow_insecure)
|
|
188
|
-
self.sock.setsockopt(zmq.LINGER, 0)
|
|
189
195
|
server_ip = gethostbyname(server_host)
|
|
190
|
-
self.sock.connect('tcp://{}:{}'.format(server_ip, server_port))
|
|
191
|
-
self.mainloop_thread = threading.Thread(target=self.mainloop)
|
|
192
|
-
self.mainloop_thread.daemon = True
|
|
193
|
-
self.mainloop_thread.start()
|
|
194
196
|
if isinstance(server_ip, bytes):
|
|
195
197
|
server_ip = server_ip.decode()
|
|
198
|
+
context = SecureContext.instance(shared_secret=shared_secret)
|
|
199
|
+
# A DEALER, not a REQ, so that a late reply is still read after the next
|
|
200
|
+
# heartbeat is sent:
|
|
201
|
+
self.sock = context.socket(zmq.DEALER, allow_insecure=allow_insecure)
|
|
202
|
+
self.sock.setsockopt(zmq.LINGER, 0)
|
|
203
|
+
self.sock.connect('tcp://{}:{}'.format(server_ip, server_port))
|
|
204
|
+
self.heartbeat_interval = heartbeat_interval
|
|
205
|
+
self.allowed_missed_heartbeats = allowed_missed_heartbeats
|
|
206
|
+
self.missed_parent_contacts = 0
|
|
196
207
|
if ipaddress.ip_address(server_ip).is_loopback:
|
|
197
208
|
self.timeout = self.LOCALHOST_TIMEOUT
|
|
198
209
|
else:
|
|
199
210
|
self.timeout = self.REMOTE_TIMEOUT
|
|
211
|
+
self.mainloop_thread = threading.Thread(target=self.mainloop)
|
|
212
|
+
self.mainloop_thread.daemon = True
|
|
213
|
+
self.mainloop_thread.start()
|
|
200
214
|
|
|
201
215
|
def mainloop(self):
|
|
202
216
|
try:
|
|
203
217
|
pid = str(os.getpid()).encode('utf8')
|
|
204
|
-
while
|
|
205
|
-
time.sleep(self.
|
|
206
|
-
self.sock.
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
218
|
+
while self.missed_parent_contacts < self.allowed_missed_heartbeats:
|
|
219
|
+
time.sleep(self.heartbeat_interval)
|
|
220
|
+
self.sock.send_multipart([b'', pid])
|
|
221
|
+
self.missed_parent_contacts += 1
|
|
222
|
+
self.sock.poll(self.timeout * 1000)
|
|
223
|
+
# Read every reply waiting. A late one still counts, so that a parent
|
|
224
|
+
# that is merely slow to reply is not taken for dead:
|
|
225
|
+
while self.sock.poll(0):
|
|
226
|
+
if self.sock.recv_multipart()[-1] == pid:
|
|
227
|
+
self.missed_parent_contacts = 0
|
|
212
228
|
if not zprocess._silent:
|
|
213
229
|
print('Heartbeat failure', file=sys.stderr)
|
|
214
230
|
os.kill(os.getpid(), signal.SIGTERM)
|
|
@@ -1259,6 +1275,8 @@ class ProcessTree(object):
|
|
|
1259
1275
|
output_redirection_port=None,
|
|
1260
1276
|
remote_process_client=None,
|
|
1261
1277
|
startup_timeout=5,
|
|
1278
|
+
heartbeat_interval=HeartbeatClient.HEARTBEAT_INTERVAL,
|
|
1279
|
+
allowed_missed_heartbeats=HeartbeatClient.DEFAULT_ALLOWED_MISSED_HEARTBEATS,
|
|
1262
1280
|
pymodule=False,
|
|
1263
1281
|
args=None,
|
|
1264
1282
|
startup_queue=None,
|
|
@@ -1325,6 +1343,8 @@ class ProcessTree(object):
|
|
|
1325
1343
|
'from_parent_port': to_child_port,
|
|
1326
1344
|
'heartbeat_server_host': 'localhost',
|
|
1327
1345
|
'heartbeat_server_port': self.heartbeat_server.port,
|
|
1346
|
+
'heartbeat_interval': heartbeat_interval,
|
|
1347
|
+
'allowed_missed_heartbeats': allowed_missed_heartbeats,
|
|
1328
1348
|
'broker_host': self.broker_host,
|
|
1329
1349
|
'broker_in_port': self.broker_in_port,
|
|
1330
1350
|
'broker_out_port': self.broker_out_port,
|
|
@@ -1442,6 +1462,14 @@ class ProcessTree(object):
|
|
|
1442
1462
|
heartbeat_server_port,
|
|
1443
1463
|
shared_secret=self.shared_secret,
|
|
1444
1464
|
allow_insecure=self.allow_insecure,
|
|
1465
|
+
heartbeat_interval=parentinfo.get(
|
|
1466
|
+
'heartbeat_interval',
|
|
1467
|
+
HeartbeatClient.HEARTBEAT_INTERVAL,
|
|
1468
|
+
),
|
|
1469
|
+
# TODO: remove legacy code in the future (introduced 2026)
|
|
1470
|
+
# Older parents will not include this field, so preserve the legacy
|
|
1471
|
+
# single-miss behavior when connecting to them.
|
|
1472
|
+
allowed_missed_heartbeats=parentinfo.get('allowed_missed_heartbeats', 1),
|
|
1445
1473
|
)
|
|
1446
1474
|
|
|
1447
1475
|
self.broker_host = parentinfo['broker_host']
|
|
@@ -187,7 +187,14 @@ def disable_quick_edit():
|
|
|
187
187
|
# Already disabled, nothing for us to do.
|
|
188
188
|
return
|
|
189
189
|
new_mode = (orig_mode | ENABLE_EXTENDED_FLAGS) & ~ENABLE_QUICK_EDIT
|
|
190
|
-
|
|
190
|
+
try:
|
|
191
|
+
console.SetConsoleMode(new_mode)
|
|
192
|
+
except pywintypes.error:
|
|
193
|
+
# The console rejects the mode. This happens when the process was relaunched
|
|
194
|
+
# from a PowerShell prompt: while PSReadLine waits for input it leaves
|
|
195
|
+
# ENABLE_ECHO_INPUT set without ENABLE_LINE_INPUT, and SetConsoleMode rejects
|
|
196
|
+
# any mode with that combination.
|
|
197
|
+
return
|
|
191
198
|
atexit.register(console.SetConsoleMode, orig_mode)
|
|
192
199
|
|
|
193
200
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: zprocess
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.28.0rc2
|
|
4
4
|
Summary: A set of utilities for multiprocessing using zeromq
|
|
5
5
|
Author-email: Christopher Billington <chrisjbillington@gmail.com>
|
|
6
6
|
License-Expression: BSD-2-Clause
|
|
@@ -11,11 +11,17 @@ Project-URL: Downloads, https://pypi.org/project/zprocess/
|
|
|
11
11
|
Project-URL: Tracker, https://github.com/chrisjbillington/zprocess/issues
|
|
12
12
|
Keywords: multiprocessing,IPC,zeromq
|
|
13
13
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
14
|
-
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
19
|
+
Requires-Python: >=3.11
|
|
15
20
|
Description-Content-Type: text/markdown
|
|
16
21
|
License-File: LICENSE.txt
|
|
17
|
-
Requires-Dist: setuptools_scm
|
|
22
|
+
Requires-Dist: setuptools_scm>=10
|
|
18
23
|
Requires-Dist: pyzmq
|
|
24
|
+
Requires-Dist: appnope; sys_platform == "darwin"
|
|
19
25
|
Requires-Dist: pywin32; sys_platform == "win32"
|
|
20
26
|
Requires-Dist: windows-curses; sys_platform == "win32"
|
|
21
27
|
Provides-Extra: test
|
|
@@ -31,6 +31,8 @@ zprocess.egg-info/PKG-INFO
|
|
|
31
31
|
zprocess.egg-info/SOURCES.txt
|
|
32
32
|
zprocess.egg-info/dependency_links.txt
|
|
33
33
|
zprocess.egg-info/requires.txt
|
|
34
|
+
zprocess.egg-info/scm_file_list.json
|
|
35
|
+
zprocess.egg-info/scm_version.json
|
|
34
36
|
zprocess.egg-info/top_level.txt
|
|
35
37
|
zprocess/examples/example_child.py
|
|
36
38
|
zprocess/examples/example_parent.py
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
{
|
|
2
|
+
"files": [
|
|
3
|
+
".github/workflows/release-vars.sh",
|
|
4
|
+
".github/workflows/release.yml",
|
|
5
|
+
".gitignore",
|
|
6
|
+
"LICENSE.txt",
|
|
7
|
+
"README.md",
|
|
8
|
+
"doc/Makefile",
|
|
9
|
+
"doc/conf.py",
|
|
10
|
+
"doc/index.rst",
|
|
11
|
+
"doc/requirements.txt",
|
|
12
|
+
"pyproject.toml",
|
|
13
|
+
"pytest.ini",
|
|
14
|
+
"requirements.txt",
|
|
15
|
+
"run_tests.py",
|
|
16
|
+
"tests/GUI_output_redirection.py",
|
|
17
|
+
"tests/output_redirection.py",
|
|
18
|
+
"tests/test_remote.py",
|
|
19
|
+
"tests/test_zlock_server.py",
|
|
20
|
+
"tests/test_zlog_server.py",
|
|
21
|
+
"tests/test_zprocess.py",
|
|
22
|
+
"zprocess/__init__.py",
|
|
23
|
+
"zprocess/__version__.py",
|
|
24
|
+
"zprocess/clientserver.py",
|
|
25
|
+
"zprocess/cryptoproxy.py",
|
|
26
|
+
"zprocess/examples/example_child.py",
|
|
27
|
+
"zprocess/examples/example_parent.py",
|
|
28
|
+
"zprocess/examples/processclass_example.py",
|
|
29
|
+
"zprocess/examples/processclass_remote_example.py",
|
|
30
|
+
"zprocess/locking/__init__.py",
|
|
31
|
+
"zprocess/locking/__main__.py",
|
|
32
|
+
"zprocess/makesecret.py",
|
|
33
|
+
"zprocess/process_class_wrapper.py",
|
|
34
|
+
"zprocess/process_tree.py",
|
|
35
|
+
"zprocess/remote/__init__.py",
|
|
36
|
+
"zprocess/remote/__main__.py",
|
|
37
|
+
"zprocess/remote/curses_server.py",
|
|
38
|
+
"zprocess/remote/server.py",
|
|
39
|
+
"zprocess/security.py",
|
|
40
|
+
"zprocess/socks/__init__.py",
|
|
41
|
+
"zprocess/socks/server.py",
|
|
42
|
+
"zprocess/tasks.py",
|
|
43
|
+
"zprocess/utils.py",
|
|
44
|
+
"zprocess/zlock/__init__.py",
|
|
45
|
+
"zprocess/zlock/__main__.py",
|
|
46
|
+
"zprocess/zlock/server.py",
|
|
47
|
+
"zprocess/zlog/__init__.py",
|
|
48
|
+
"zprocess/zlog/__main__.py",
|
|
49
|
+
"zprocess/zlog/server.py"
|
|
50
|
+
]
|
|
51
|
+
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|