parsl 2024.1.29__py3-none-any.whl → 2024.2.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. parsl/curvezmq.py +205 -0
  2. parsl/dataflow/dflow.py +1 -1
  3. parsl/executors/high_throughput/executor.py +78 -49
  4. parsl/executors/high_throughput/interchange.py +14 -7
  5. parsl/executors/high_throughput/process_worker_pool.py +15 -7
  6. parsl/executors/high_throughput/zmq_pipes.py +21 -15
  7. parsl/executors/taskvine/manager.py +44 -43
  8. parsl/monitoring/monitoring.py +18 -3
  9. parsl/providers/errors.py +4 -6
  10. parsl/providers/slurm/slurm.py +7 -6
  11. parsl/tests/configs/ad_hoc_cluster_htex.py +1 -0
  12. parsl/tests/configs/azure_single_node.py +1 -0
  13. parsl/tests/configs/bluewaters.py +1 -0
  14. parsl/tests/configs/bridges.py +1 -0
  15. parsl/tests/configs/cc_in2p3.py +1 -0
  16. parsl/tests/configs/comet.py +1 -0
  17. parsl/tests/configs/cooley_htex.py +1 -0
  18. parsl/tests/configs/ec2_single_node.py +1 -0
  19. parsl/tests/configs/ec2_spot.py +1 -0
  20. parsl/tests/configs/frontera.py +1 -0
  21. parsl/tests/configs/htex_ad_hoc_cluster.py +1 -0
  22. parsl/tests/configs/htex_local.py +1 -0
  23. parsl/tests/configs/htex_local_alternate.py +1 -0
  24. parsl/tests/configs/htex_local_intask_staging.py +1 -0
  25. parsl/tests/configs/htex_local_rsync_staging.py +1 -0
  26. parsl/tests/configs/local_adhoc.py +1 -0
  27. parsl/tests/configs/midway.py +1 -0
  28. parsl/tests/configs/nscc_singapore.py +1 -0
  29. parsl/tests/configs/osg_htex.py +1 -0
  30. parsl/tests/configs/petrelkube.py +1 -0
  31. parsl/tests/configs/summit.py +1 -0
  32. parsl/tests/configs/swan_htex.py +1 -0
  33. parsl/tests/configs/theta.py +1 -0
  34. parsl/tests/manual_tests/htex_local.py +1 -0
  35. parsl/tests/manual_tests/test_ad_hoc_htex.py +1 -0
  36. parsl/tests/manual_tests/test_fan_in_out_htex_remote.py +1 -0
  37. parsl/tests/manual_tests/test_memory_limits.py +1 -0
  38. parsl/tests/scaling_tests/htex_local.py +1 -0
  39. parsl/tests/sites/test_affinity.py +1 -0
  40. parsl/tests/sites/test_concurrent.py +2 -1
  41. parsl/tests/sites/test_dynamic_executor.py +1 -0
  42. parsl/tests/sites/test_worker_info.py +1 -0
  43. parsl/tests/test_bash_apps/test_stdout.py +6 -1
  44. parsl/tests/test_curvezmq.py +455 -0
  45. parsl/tests/test_data/test_file_apps.py +5 -5
  46. parsl/tests/test_data/test_file_staging.py +3 -3
  47. parsl/tests/test_docs/test_kwargs.py +3 -3
  48. parsl/tests/test_htex/test_htex.py +46 -0
  49. parsl/tests/test_htex/test_htex_zmq_binding.py +53 -13
  50. parsl/tests/test_python_apps/test_futures.py +5 -5
  51. parsl/tests/test_regression/test_97_parallelism_0.py +1 -0
  52. parsl/tests/test_scaling/test_block_error_handler.py +6 -5
  53. parsl/tests/test_scaling/test_regression_1621.py +1 -0
  54. parsl/tests/test_scaling/test_scale_down.py +1 -0
  55. parsl/version.py +1 -1
  56. {parsl-2024.1.29.data → parsl-2024.2.5.data}/scripts/process_worker_pool.py +15 -7
  57. {parsl-2024.1.29.dist-info → parsl-2024.2.5.dist-info}/METADATA +3 -3
  58. {parsl-2024.1.29.dist-info → parsl-2024.2.5.dist-info}/RECORD +64 -61
  59. {parsl-2024.1.29.data → parsl-2024.2.5.data}/scripts/exec_parsl_function.py +0 -0
  60. {parsl-2024.1.29.data → parsl-2024.2.5.data}/scripts/parsl_coprocess.py +0 -0
  61. {parsl-2024.1.29.dist-info → parsl-2024.2.5.dist-info}/LICENSE +0 -0
  62. {parsl-2024.1.29.dist-info → parsl-2024.2.5.dist-info}/WHEEL +0 -0
  63. {parsl-2024.1.29.dist-info → parsl-2024.2.5.dist-info}/entry_points.txt +0 -0
  64. {parsl-2024.1.29.dist-info → parsl-2024.2.5.dist-info}/top_level.txt +0 -0
@@ -60,6 +60,7 @@ def test_dynamic_executor():
60
60
  label='htex_local',
61
61
  cores_per_worker=1,
62
62
  max_workers=5,
63
+ encrypted=True,
63
64
  provider=LocalProvider(
64
65
  init_blocks=1,
65
66
  max_blocks=1,
@@ -15,6 +15,7 @@ def local_config():
15
15
  label="htex_Local",
16
16
  worker_debug=True,
17
17
  max_workers=4,
18
+ encrypted=True,
18
19
  provider=LocalProvider(
19
20
  channel=LocalChannel(),
20
21
  init_blocks=1,
@@ -1,6 +1,7 @@
1
1
  import os
2
2
 
3
3
  import pytest
4
+ import typeguard
4
5
 
5
6
  import parsl.app.errors as perror
6
7
  from parsl.app.app import bash_app
@@ -44,7 +45,11 @@ def test_bad_stdout_specs(spec):
44
45
  try:
45
46
  fn.result()
46
47
  except Exception as e:
47
- assert isinstance(e, TypeError) or isinstance(e, perror.BadStdStreamFile), "Exception is wrong type"
48
+ # This tests for TypeCheckError by string matching on the type name
49
+ # because that class does not exist in typeguard 2.x - it is new in
50
+ # typeguard 4.x. When typeguard 2.x support is dropped, this test can
51
+ # become an isinstance check.
52
+ assert "TypeCheckError" in str(type(e)) or isinstance(e, TypeError) or isinstance(e, perror.BadStdStreamFile), "Exception is wrong type"
48
53
  else:
49
54
  assert False, "Did not raise expected exception"
50
55
 
@@ -0,0 +1,455 @@
1
+ import os
2
+ import pathlib
3
+ from typing import Union
4
+ from unittest import mock
5
+
6
+ import pytest
7
+ import zmq
8
+ import zmq.auth
9
+ from zmq.auth.thread import ThreadAuthenticator
10
+
11
+ from parsl import curvezmq
12
+
13
+ ADDR = "tcp://127.0.0.1"
14
+
15
+
16
+ def get_server_socket(ctx: curvezmq.ServerContext):
17
+ sock = ctx.socket(zmq.PULL)
18
+ sock.setsockopt(zmq.RCVTIMEO, 200)
19
+ sock.setsockopt(zmq.LINGER, 0)
20
+ port = sock.bind_to_random_port(ADDR)
21
+ return sock, port
22
+
23
+
24
+ def get_client_socket(ctx: curvezmq.ClientContext, port: int):
25
+ sock = ctx.socket(zmq.PUSH)
26
+ sock.setsockopt(zmq.SNDTIMEO, 200)
27
+ sock.setsockopt(zmq.LINGER, 0)
28
+ sock.connect(f"{ADDR}:{port}")
29
+ return sock
30
+
31
+
32
+ def get_external_server_socket(
33
+ ctx: Union[curvezmq.ServerContext, zmq.Context], secret_key: bytes
34
+ ):
35
+ sock = ctx.socket(zmq.PULL)
36
+ sock.setsockopt(zmq.RCVTIMEO, 200)
37
+ sock.setsockopt(zmq.LINGER, 0)
38
+ sock.setsockopt(zmq.CURVE_SECRETKEY, secret_key)
39
+ sock.setsockopt(zmq.CURVE_SERVER, True)
40
+ port = sock.bind_to_random_port(ADDR)
41
+ return sock, port
42
+
43
+
44
+ def get_external_client_socket(
45
+ ctx: Union[curvezmq.ClientContext, zmq.Context],
46
+ public_key: bytes,
47
+ secret_key: bytes,
48
+ server_key: bytes,
49
+ port: int,
50
+ ):
51
+ sock = ctx.socket(zmq.PUSH)
52
+ sock.setsockopt(zmq.LINGER, 0)
53
+ sock.setsockopt(zmq.CURVE_PUBLICKEY, public_key)
54
+ sock.setsockopt(zmq.CURVE_SECRETKEY, secret_key)
55
+ sock.setsockopt(zmq.CURVE_SERVERKEY, server_key)
56
+ sock.connect(f"{ADDR}:{port}")
57
+ return sock
58
+
59
+
60
+ @pytest.fixture
61
+ def encrypted(request: pytest.FixtureRequest):
62
+ if hasattr(request, "param"):
63
+ return request.param
64
+ return True
65
+
66
+
67
+ @pytest.fixture
68
+ def cert_dir(encrypted: bool, tmpd_cwd: pathlib.Path):
69
+ if not encrypted:
70
+ return None
71
+ return curvezmq.create_certificates(tmpd_cwd)
72
+
73
+
74
+ @pytest.fixture
75
+ def server_ctx(cert_dir: Union[str, None]):
76
+ ctx = curvezmq.ServerContext(cert_dir)
77
+ yield ctx
78
+ ctx.destroy()
79
+
80
+
81
+ @pytest.fixture
82
+ def client_ctx(cert_dir: Union[str, None]):
83
+ ctx = curvezmq.ClientContext(cert_dir)
84
+ yield ctx
85
+ ctx.destroy()
86
+
87
+
88
+ @pytest.fixture
89
+ def zmq_ctx():
90
+ ctx = zmq.Context()
91
+ yield ctx
92
+ ctx.destroy()
93
+
94
+
95
+ @pytest.mark.local
96
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
97
+ def test_client_context_init(cert_dir: Union[str, None]):
98
+ ctx = curvezmq.ClientContext(cert_dir=cert_dir)
99
+
100
+ assert ctx.cert_dir == cert_dir
101
+ if cert_dir is None:
102
+ assert not ctx.encrypted
103
+ else:
104
+ assert ctx.encrypted
105
+
106
+ ctx.destroy()
107
+
108
+
109
+ @pytest.mark.local
110
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
111
+ def test_server_context_init(cert_dir: Union[str, None]):
112
+ ctx = curvezmq.ServerContext(cert_dir=cert_dir)
113
+
114
+ assert ctx.cert_dir == cert_dir
115
+ if cert_dir is None:
116
+ assert not ctx.encrypted
117
+ assert not ctx.auth_thread
118
+ else:
119
+ assert ctx.encrypted
120
+ assert isinstance(ctx.auth_thread, ThreadAuthenticator)
121
+
122
+ ctx.destroy()
123
+
124
+
125
+ @pytest.mark.local
126
+ def test_create_certificates(tmpd_cwd: pathlib.Path):
127
+ cert_dir = tmpd_cwd / "certificates"
128
+ assert not os.path.exists(cert_dir)
129
+
130
+ ret = curvezmq.create_certificates(tmpd_cwd)
131
+
132
+ assert str(cert_dir) == ret
133
+ assert os.path.exists(cert_dir)
134
+ assert os.stat(cert_dir).st_mode & 0o777 == 0o700
135
+ assert len(os.listdir(cert_dir)) == 4
136
+
137
+
138
+ @pytest.mark.local
139
+ def test_create_certificates_overwrite(tmpd_cwd: pathlib.Path):
140
+ cert_dir = curvezmq.create_certificates(tmpd_cwd)
141
+ client_pub_1, client_sec_1 = curvezmq._load_certificate(cert_dir, name="client")
142
+ server_pub_1, server_sec_1 = curvezmq._load_certificate(cert_dir, name="server")
143
+
144
+ curvezmq.create_certificates(tmpd_cwd)
145
+ client_pub_2, client_sec_2 = curvezmq._load_certificate(cert_dir, name="client")
146
+ server_pub_2, server_sec_2 = curvezmq._load_certificate(cert_dir, name="server")
147
+
148
+ assert client_pub_1 != client_pub_2
149
+ assert client_sec_1 != client_sec_2
150
+ assert server_pub_1 != server_pub_2
151
+ assert server_sec_1 != server_sec_2
152
+
153
+
154
+ @pytest.mark.local
155
+ def test_cert_dir_not_private(tmpd_cwd: pathlib.Path):
156
+ cert_dir = tmpd_cwd / "certificates"
157
+ os.makedirs(cert_dir, mode=0o777)
158
+ client_ctx = curvezmq.ClientContext(cert_dir)
159
+ server_ctx = curvezmq.ServerContext(cert_dir)
160
+
161
+ err_msg = "directory must be private"
162
+
163
+ with pytest.raises(OSError) as pyt_e:
164
+ client_ctx.socket(zmq.REQ)
165
+ assert err_msg in str(pyt_e.value)
166
+
167
+ with pytest.raises(OSError) as pyt_e:
168
+ server_ctx.socket(zmq.REP)
169
+ assert err_msg in str(pyt_e.value)
170
+
171
+ client_ctx.destroy()
172
+ server_ctx.destroy()
173
+
174
+
175
+ @pytest.mark.local
176
+ def test_missing_cert_dir():
177
+ cert_dir = "/bad/cert/dir"
178
+ client_ctx = curvezmq.ClientContext(cert_dir)
179
+ server_ctx = curvezmq.ServerContext(cert_dir)
180
+
181
+ err_msg = "No such file or directory"
182
+
183
+ with pytest.raises(FileNotFoundError) as pyt_e:
184
+ client_ctx.socket(zmq.REQ)
185
+ assert err_msg in str(pyt_e.value)
186
+
187
+ with pytest.raises(FileNotFoundError) as pyt_e:
188
+ server_ctx.socket(zmq.REP)
189
+ assert err_msg in str(pyt_e.value)
190
+
191
+ client_ctx.destroy()
192
+ server_ctx.destroy()
193
+
194
+
195
+ @pytest.mark.local
196
+ def test_missing_secret_file(tmpd_cwd: pathlib.Path):
197
+ cert_dir = tmpd_cwd / "certificates"
198
+ os.makedirs(cert_dir, mode=0o700)
199
+
200
+ client_ctx = curvezmq.ClientContext(cert_dir)
201
+ server_ctx = curvezmq.ServerContext(cert_dir)
202
+
203
+ err_msg = "Invalid certificate file"
204
+
205
+ with pytest.raises(OSError) as pyt_e:
206
+ client_ctx.socket(zmq.REQ)
207
+ assert err_msg in str(pyt_e.value)
208
+
209
+ with pytest.raises(OSError) as pyt_e:
210
+ server_ctx.socket(zmq.REP)
211
+ assert err_msg in str(pyt_e.value)
212
+
213
+ client_ctx.destroy()
214
+ server_ctx.destroy()
215
+
216
+
217
+ @pytest.mark.local
218
+ def test_bad_secret_file(tmpd_cwd: pathlib.Path):
219
+ client_sec = tmpd_cwd / "client.key_secret"
220
+ server_sec = tmpd_cwd / "server.key_secret"
221
+ client_sec.write_text("bad")
222
+ server_sec.write_text("boy")
223
+
224
+ client_ctx = curvezmq.ClientContext(tmpd_cwd)
225
+ server_ctx = curvezmq.ServerContext(tmpd_cwd)
226
+
227
+ err_msg = "No public key found"
228
+
229
+ with pytest.raises(ValueError) as pyt_e:
230
+ client_ctx.socket(zmq.REQ)
231
+ assert err_msg in str(pyt_e.value)
232
+
233
+ with pytest.raises(ValueError) as pyt_e:
234
+ server_ctx.socket(zmq.REP)
235
+ assert err_msg in str(pyt_e.value)
236
+
237
+ client_ctx.destroy()
238
+ server_ctx.destroy()
239
+
240
+
241
+ @pytest.mark.local
242
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
243
+ def test_client_context_term(client_ctx: curvezmq.ClientContext):
244
+ assert not client_ctx.closed
245
+
246
+ client_ctx.term()
247
+
248
+ assert client_ctx.closed
249
+
250
+
251
+ @pytest.mark.local
252
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
253
+ def test_server_context_term(server_ctx: curvezmq.ServerContext, encrypted: bool):
254
+ assert not server_ctx.closed
255
+ if encrypted:
256
+ assert server_ctx.auth_thread
257
+ assert server_ctx.auth_thread.pipe
258
+
259
+ server_ctx.term()
260
+
261
+ assert server_ctx.closed
262
+ if encrypted:
263
+ assert server_ctx.auth_thread
264
+ assert not server_ctx.auth_thread.pipe
265
+
266
+
267
+ @pytest.mark.local
268
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
269
+ def test_client_context_destroy(client_ctx: curvezmq.ClientContext):
270
+ sock = client_ctx.socket(zmq.REP)
271
+
272
+ assert not client_ctx.closed
273
+
274
+ client_ctx.destroy()
275
+
276
+ assert sock.closed
277
+ assert client_ctx.closed
278
+
279
+
280
+ @pytest.mark.local
281
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
282
+ def test_server_context_destroy(server_ctx: curvezmq.ServerContext, encrypted: bool):
283
+ sock = server_ctx.socket(zmq.REP)
284
+
285
+ assert not server_ctx.closed
286
+ if encrypted:
287
+ assert server_ctx.auth_thread
288
+ assert server_ctx.auth_thread.pipe
289
+
290
+ server_ctx.destroy()
291
+
292
+ assert sock.closed
293
+ assert server_ctx.closed
294
+ if encrypted:
295
+ assert server_ctx.auth_thread
296
+ assert not server_ctx.auth_thread.pipe
297
+
298
+
299
+ @pytest.mark.local
300
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
301
+ def test_client_context_recreate(client_ctx: curvezmq.ClientContext):
302
+ hidden_ctx = client_ctx._ctx
303
+ sock = client_ctx.socket(zmq.REQ)
304
+
305
+ assert not sock.closed
306
+ assert not client_ctx.closed
307
+
308
+ client_ctx.recreate()
309
+
310
+ assert sock.closed
311
+ assert not client_ctx.closed
312
+ assert hidden_ctx != client_ctx._ctx
313
+ assert hidden_ctx.closed
314
+
315
+
316
+ @pytest.mark.local
317
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
318
+ def test_server_context_recreate(server_ctx: curvezmq.ServerContext, encrypted: bool):
319
+ hidden_ctx = server_ctx._ctx
320
+ sock = server_ctx.socket(zmq.REP)
321
+
322
+ assert not sock.closed
323
+ assert not server_ctx.closed
324
+ if encrypted:
325
+ assert server_ctx.auth_thread
326
+ auth_thread = server_ctx.auth_thread
327
+ assert auth_thread.pipe
328
+
329
+ server_ctx.recreate()
330
+
331
+ assert sock.closed
332
+ assert not server_ctx.closed
333
+ assert hidden_ctx.closed
334
+ assert hidden_ctx != server_ctx._ctx
335
+ if encrypted:
336
+ assert server_ctx.auth_thread
337
+ assert auth_thread != server_ctx.auth_thread
338
+ assert server_ctx.auth_thread.pipe
339
+
340
+
341
+ @pytest.mark.local
342
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
343
+ def test_connection(
344
+ server_ctx: curvezmq.ServerContext, client_ctx: curvezmq.ClientContext
345
+ ):
346
+ server_socket, port = get_server_socket(server_ctx)
347
+ client_socket = get_client_socket(client_ctx, port)
348
+
349
+ msg = b"howdy"
350
+ client_socket.send(msg)
351
+ recv = server_socket.recv()
352
+
353
+ assert recv == msg
354
+
355
+
356
+ @pytest.mark.local
357
+ @mock.patch.object(curvezmq, "_load_certificate")
358
+ def test_invalid_key_format(
359
+ mock_load_cert: mock.MagicMock,
360
+ server_ctx: curvezmq.ServerContext,
361
+ client_ctx: curvezmq.ClientContext,
362
+ ):
363
+ mock_load_cert.return_value = (b"badkey", b"badkey")
364
+
365
+ with pytest.raises(ValueError) as e1_info:
366
+ server_ctx.socket(zmq.REP)
367
+ with pytest.raises(ValueError) as e2_info:
368
+ client_ctx.socket(zmq.REQ)
369
+ e1, e2 = e1_info.exconly, e2_info.exconly
370
+
371
+ assert str(e1) == str(e2)
372
+ assert "Invalid CurveZMQ key format" in str(e1)
373
+
374
+
375
+ @pytest.mark.local
376
+ def test_invalid_client_keys(server_ctx: curvezmq.ServerContext, zmq_ctx: zmq.Context):
377
+ server_socket, port = get_server_socket(server_ctx)
378
+
379
+ cert_dir = server_ctx.cert_dir
380
+ assert cert_dir # For mypy
381
+ public_key, secret_key = curvezmq._load_certificate(cert_dir, "client")
382
+ server_key, _ = curvezmq._load_certificate(cert_dir, "server")
383
+
384
+ BAD_PUB_KEY, BAD_SEC_KEY = zmq.curve_keypair()
385
+ msg = b"howdy"
386
+
387
+ client_socket = get_external_client_socket(
388
+ zmq_ctx,
389
+ public_key,
390
+ secret_key,
391
+ server_key,
392
+ port,
393
+ )
394
+ client_socket.send(msg)
395
+ assert server_socket.recv() == msg
396
+
397
+ client_socket = get_external_client_socket(
398
+ zmq_ctx,
399
+ BAD_PUB_KEY,
400
+ BAD_SEC_KEY,
401
+ server_key,
402
+ port,
403
+ )
404
+ client_socket.send(msg)
405
+ with pytest.raises(zmq.Again):
406
+ server_socket.recv()
407
+
408
+ client_socket = get_external_client_socket(
409
+ zmq_ctx,
410
+ public_key,
411
+ secret_key,
412
+ BAD_PUB_KEY,
413
+ port,
414
+ )
415
+ client_socket.send(msg)
416
+ with pytest.raises(zmq.Again):
417
+ server_socket.recv()
418
+
419
+ # Ensure sockets are operational
420
+ client_socket = get_external_client_socket(
421
+ zmq_ctx,
422
+ public_key,
423
+ secret_key,
424
+ server_key,
425
+ port,
426
+ )
427
+ client_socket.send(msg)
428
+ assert server_socket.recv() == msg
429
+
430
+
431
+ @pytest.mark.local
432
+ def test_invalid_server_key(client_ctx: curvezmq.ClientContext, zmq_ctx: zmq.Context):
433
+ cert_dir = client_ctx.cert_dir
434
+ assert cert_dir # For mypy
435
+ _, secret_key = curvezmq._load_certificate(cert_dir, "server")
436
+
437
+ _, BAD_SEC_KEY = zmq.curve_keypair()
438
+ msg = b"howdy"
439
+
440
+ server_socket, port = get_external_server_socket(zmq_ctx, secret_key)
441
+ client_socket = get_client_socket(client_ctx, port)
442
+ client_socket.send(msg)
443
+ assert server_socket.recv() == msg
444
+
445
+ server_socket, port = get_external_server_socket(zmq_ctx, BAD_SEC_KEY)
446
+ client_socket = get_client_socket(client_ctx, port)
447
+ client_socket.send(msg)
448
+ with pytest.raises(zmq.Again):
449
+ server_socket.recv()
450
+
451
+ # Ensure sockets are operational
452
+ server_socket, port = get_external_server_socket(zmq_ctx, secret_key)
453
+ client_socket = get_client_socket(client_ctx, port)
454
+ client_socket.send(msg)
455
+ assert server_socket.recv() == msg
@@ -35,10 +35,10 @@ def increment(inputs=(), outputs=(), stdout=None, stderr=None):
35
35
 
36
36
 
37
37
  @pytest.mark.staging_required
38
- def test_increment(tmp_path, depth=5):
38
+ def test_increment(tmpd_cwd, depth=5):
39
39
  """Test simple pipeline A->B...->N"""
40
40
  # Test setup
41
- first_fpath = tmp_path / "test0.txt"
41
+ first_fpath = tmpd_cwd / "test0.txt"
42
42
  first_fpath.write_text("0\n")
43
43
 
44
44
  prev = [File(first_fpath)]
@@ -46,9 +46,9 @@ def test_increment(tmp_path, depth=5):
46
46
  for i in range(1, depth):
47
47
  f = increment(
48
48
  inputs=prev,
49
- outputs=[File(tmp_path / f"test{i}.txt")],
50
- stdout=tmp_path / f"incr{i}.out",
51
- stderr=tmp_path / f"incr{i}.err",
49
+ outputs=[File(tmpd_cwd / f"test{i}.txt")],
50
+ stdout=tmpd_cwd / f"incr{i}.out",
51
+ stderr=tmpd_cwd / f"incr{i}.err",
52
52
  )
53
53
  prev = f.outputs
54
54
  futs.append((i, prev[0]))
@@ -11,10 +11,10 @@ def cat(inputs=(), outputs=(), stdout=None, stderr=None):
11
11
 
12
12
 
13
13
  @pytest.mark.staging_required
14
- def test_regression_200(tmp_path):
14
+ def test_regression_200(tmpd_cwd):
15
15
  """Regression test for #200. Pickleablility of Files"""
16
- opath = tmp_path / "test_output.txt"
17
- fpath = tmp_path / "test.txt"
16
+ opath = tmpd_cwd / "test_output.txt"
17
+ fpath = tmpd_cwd / "test.txt"
18
18
 
19
19
  fpath.write_text("Hello World")
20
20
  f = cat(inputs=[File(fpath)], outputs=[File(opath)])
@@ -19,7 +19,7 @@ def test_inputs():
19
19
  assert reduce_future.result() == 6
20
20
 
21
21
 
22
- def test_outputs(tmpdir):
22
+ def test_outputs(tmpd_cwd):
23
23
  @python_app()
24
24
  def write_app(message, outputs=()):
25
25
  """Write a single message to every file in outputs"""
@@ -28,8 +28,8 @@ def test_outputs(tmpdir):
28
28
  print(message, file=fp)
29
29
 
30
30
  to_write = [
31
- File(Path(tmpdir) / 'output-0.txt'),
32
- File(Path(tmpdir) / 'output-1.txt')
31
+ File(Path(tmpd_cwd) / 'output-0.txt'),
32
+ File(Path(tmpd_cwd) / 'output-1.txt')
33
33
  ]
34
34
  write_app('Hello!', outputs=to_write).result()
35
35
  for path in to_write:
@@ -0,0 +1,46 @@
1
+ import pathlib
2
+
3
+ import pytest
4
+
5
+ from parsl import curvezmq
6
+ from parsl import HighThroughputExecutor
7
+
8
+
9
+ @pytest.mark.local
10
+ @pytest.mark.parametrize("encrypted", (True, False))
11
+ @pytest.mark.parametrize("cert_dir_provided", (True, False))
12
+ def test_htex_start_encrypted(
13
+ encrypted: bool, cert_dir_provided: bool, tmpd_cwd: pathlib.Path
14
+ ):
15
+ htex = HighThroughputExecutor(encrypted=encrypted)
16
+ htex.run_dir = str(tmpd_cwd)
17
+ if cert_dir_provided:
18
+ provided_base_dir = tmpd_cwd / "provided"
19
+ provided_base_dir.mkdir()
20
+ cert_dir = curvezmq.create_certificates(provided_base_dir)
21
+ htex.cert_dir = cert_dir
22
+ else:
23
+ cert_dir = curvezmq.create_certificates(htex.logdir)
24
+
25
+ if not encrypted and cert_dir_provided:
26
+ with pytest.raises(AttributeError) as pyt_e:
27
+ htex.start()
28
+ assert "change cert_dir to None" in str(pyt_e.value)
29
+ return
30
+
31
+ htex.start()
32
+
33
+ assert htex.encrypted is encrypted
34
+ if encrypted:
35
+ assert htex.cert_dir == cert_dir
36
+ assert htex.outgoing_q.zmq_context.cert_dir == cert_dir
37
+ assert htex.incoming_q.zmq_context.cert_dir == cert_dir
38
+ assert htex.command_client.zmq_context.cert_dir == cert_dir
39
+ assert isinstance(htex.outgoing_q.zmq_context, curvezmq.ClientContext)
40
+ assert isinstance(htex.incoming_q.zmq_context, curvezmq.ClientContext)
41
+ assert isinstance(htex.command_client.zmq_context, curvezmq.ClientContext)
42
+ else:
43
+ assert htex.cert_dir is None
44
+ assert htex.outgoing_q.zmq_context.cert_dir is None
45
+ assert htex.incoming_q.zmq_context.cert_dir is None
46
+ assert htex.command_client.zmq_context.cert_dir is None
@@ -1,42 +1,82 @@
1
- import logging
1
+ import pathlib
2
+ from typing import Optional
3
+ from unittest import mock
2
4
 
3
5
  import psutil
4
6
  import pytest
5
7
  import zmq
6
8
 
9
+ from parsl import curvezmq
7
10
  from parsl.executors.high_throughput.interchange import Interchange
8
11
 
9
12
 
10
- def test_interchange_binding_no_address():
11
- ix = Interchange()
13
+ @pytest.fixture
14
+ def encrypted(request: pytest.FixtureRequest):
15
+ if hasattr(request, "param"):
16
+ return request.param
17
+ return True
18
+
19
+
20
+ @pytest.fixture
21
+ def cert_dir(encrypted: bool, tmpd_cwd: pathlib.Path):
22
+ if not encrypted:
23
+ return None
24
+ return curvezmq.create_certificates(tmpd_cwd)
25
+
26
+
27
+ @pytest.mark.local
28
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
29
+ @mock.patch.object(curvezmq.ServerContext, "socket", return_value=mock.MagicMock())
30
+ def test_interchange_curvezmq_sockets(
31
+ mock_socket: mock.MagicMock, cert_dir: Optional[str], encrypted: bool
32
+ ):
33
+ address = "127.0.0.1"
34
+ ix = Interchange(interchange_address=address, cert_dir=cert_dir)
35
+ assert isinstance(ix.zmq_context, curvezmq.ServerContext)
36
+ assert ix.zmq_context.encrypted is encrypted
37
+ assert mock_socket.call_count == 5
38
+
39
+
40
+ @pytest.mark.local
41
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
42
+ def test_interchange_binding_no_address(cert_dir: Optional[str]):
43
+ ix = Interchange(cert_dir=cert_dir)
12
44
  assert ix.interchange_address == "*"
13
45
 
14
46
 
15
- def test_interchange_binding_with_address():
47
+ @pytest.mark.local
48
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
49
+ def test_interchange_binding_with_address(cert_dir: Optional[str]):
16
50
  # Using loopback address
17
51
  address = "127.0.0.1"
18
- ix = Interchange(interchange_address=address)
52
+ ix = Interchange(interchange_address=address, cert_dir=cert_dir)
19
53
  assert ix.interchange_address == address
20
54
 
21
55
 
22
- def test_interchange_binding_with_non_ipv4_address():
56
+ @pytest.mark.local
57
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
58
+ def test_interchange_binding_with_non_ipv4_address(cert_dir: Optional[str]):
23
59
  # Confirm that a ipv4 address is required
24
60
  address = "localhost"
25
61
  with pytest.raises(zmq.error.ZMQError):
26
- Interchange(interchange_address=address)
62
+ Interchange(interchange_address=address, cert_dir=cert_dir)
27
63
 
28
64
 
29
- def test_interchange_binding_bad_address():
30
- """ Confirm that we raise a ZMQError when a bad address is supplied"""
65
+ @pytest.mark.local
66
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
67
+ def test_interchange_binding_bad_address(cert_dir: Optional[str]):
68
+ """Confirm that we raise a ZMQError when a bad address is supplied"""
31
69
  address = "550.0.0.0"
32
70
  with pytest.raises(zmq.error.ZMQError):
33
- Interchange(interchange_address=address)
71
+ Interchange(interchange_address=address, cert_dir=cert_dir)
34
72
 
35
73
 
36
- def test_limited_interface_binding():
37
- """ When address is specified the worker_port would be bound to it rather than to 0.0.0.0"""
74
+ @pytest.mark.local
75
+ @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
76
+ def test_limited_interface_binding(cert_dir: Optional[str]):
77
+ """When address is specified the worker_port would be bound to it rather than to 0.0.0.0"""
38
78
  address = "127.0.0.1"
39
- ix = Interchange(interchange_address=address)
79
+ ix = Interchange(interchange_address=address, cert_dir=cert_dir)
40
80
  ix.worker_result_port
41
81
  proc = psutil.Process()
42
82
  conns = proc.connections(kind="tcp")