parsl 2024.1.22__py3-none-any.whl → 2024.2.5__py3-none-any.whl

parsl/tests/conftest.py

@@ -48,6 +48,12 @@ def pytest_sessionstart(session):
 
 @pytest.fixture(scope="session")
 def tmpd_cwd_session(pytestconfig):
+    def _chmod(path: pathlib.Path, mode: int):
+        try:
+            path.lchmod(mode)  # support BSD and derivatives
+        except NotImplementedError:
+            path.chmod(mode)
+
     config = re.sub(r"[^A-z0-9_-]+", "_", pytestconfig.getoption('config')[0])
     cwd = pathlib.Path(os.getcwd())
     pytest_dir = cwd / ".pytest"
@@ -76,9 +82,9 @@ def tmpd_cwd_session(pytestconfig):
         for root, subdirnames, fnames in os.walk(run_to_remove):
             rpath = pathlib.Path(root)
             for d in subdirnames:
-                (rpath / d).lchmod(0o700)
+                _chmod(rpath / d, 0o700)
             for f in fnames:
-                (rpath / f).lchmod(0o600)
+                _chmod(rpath / f, 0o600)
         shutil.rmtree(run_to_remove)
 
 
@@ -148,6 +154,10 @@ def pytest_configure(config):
         'markers',
         'sshd_required: Marks tests that require a SSHD'
     )
+    config.addinivalue_line(
+        'markers',
+        'multiple_cores_required: Marks tests that require multiple cores, such as htex affinity'
+    )
 
 
 @pytest.fixture(autouse=True, scope='session')

parsl/tests/manual_tests/htex_local.py

@@ -13,6 +13,7 @@ config = Config(
             label="htex_local",
             # worker_debug=True,
             cores_per_worker=1,
+            encrypted=True,
             provider=LocalProvider(
                 channel=LocalChannel(),
                 init_blocks=1,

parsl/tests/manual_tests/test_ad_hoc_htex.py

@@ -15,6 +15,7 @@ config = Config(
             label='AdHoc',
             max_workers=2,
             worker_logdir_root="/scratch/midway2/yadunand/parsl_scripts",
+            encrypted=True,
             provider=AdHocProvider(
                 worker_init="source /scratch/midway2/yadunand/parsl_env_setup.sh",
                 channels=[SSHChannel(hostname=m,

parsl/tests/manual_tests/test_fan_in_out_htex_remote.py

@@ -16,6 +16,7 @@ def local_setup():
                 label="theta_htex",
                 # worker_debug=True,
                 cores_per_worker=4,
+                encrypted=True,
                 provider=CobaltProvider(
                     queue='debug-flat-quad',
                     account='CSC249ADCD01',

parsl/tests/manual_tests/test_memory_limits.py

@@ -28,6 +28,7 @@ def test_simple(mem_per_worker):
                 mem_per_worker=mem_per_worker,
                 cores_per_worker=0.1,
                 suppress_failure=True,
+                encrypted=True,
                 provider=LocalProvider(
                     channel=LocalChannel(),
                     init_blocks=1,

parsl/tests/scaling_tests/htex_local.py

@@ -10,6 +10,7 @@ config = Config(
             label="htex_local",
             cores_per_worker=1,
             max_workers=8,
+            encrypted=True,
             provider=LocalProvider(
                 channel=LocalChannel(),
                 init_blocks=1,

parsl/tests/sites/test_affinity.py

@@ -18,6 +18,7 @@ def local_config():
                 max_workers=2,
                 cpu_affinity='block',
                 available_accelerators=2,
+                encrypted=True,
                 provider=LocalProvider(
                     channel=LocalChannel(),
                     init_blocks=1,

parsl/tests/sites/test_concurrent.py

@@ -17,6 +17,7 @@ def make_config():
                 max_workers=2,
                 heartbeat_period=2,
                 heartbeat_threshold=4,
+                encrypted=True,
             )
         ],
         strategy='none',
@@ -24,7 +25,7 @@ def make_config():
 
 
 @mark.local
-def test_executor(tmpdir):
+def test_executor():
     my_config = make_config()
 
     with ParslPoolExecutor(my_config) as exc:

parsl/tests/sites/test_dynamic_executor.py

@@ -60,6 +60,7 @@ def test_dynamic_executor():
             label='htex_local',
             cores_per_worker=1,
             max_workers=5,
+            encrypted=True,
             provider=LocalProvider(
                 init_blocks=1,
                 max_blocks=1,

parsl/tests/sites/test_worker_info.py

@@ -15,6 +15,7 @@ def local_config():
                 label="htex_Local",
                 worker_debug=True,
                 max_workers=4,
+                encrypted=True,
                 provider=LocalProvider(
                     channel=LocalChannel(),
                     init_blocks=1,

parsl/tests/test_bash_apps/test_stdout.py

@@ -1,6 +1,7 @@
 import os
 
 import pytest
+import typeguard
 
 import parsl.app.errors as perror
 from parsl.app.app import bash_app
@@ -44,7 +45,11 @@ def test_bad_stdout_specs(spec):
     try:
         fn.result()
     except Exception as e:
-        assert isinstance(e, TypeError) or isinstance(e, perror.BadStdStreamFile), "Exception is wrong type"
+        # This tests for TypeCheckError by string matching on the type name
+        # because that class does not exist in typeguard 2.x - it is new in
+        # typeguard 4.x. When typeguard 2.x support is dropped, this test can
+        # become an isinstance check.
+        assert "TypeCheckError" in str(type(e)) or isinstance(e, TypeError) or isinstance(e, perror.BadStdStreamFile), "Exception is wrong type"
     else:
         assert False, "Did not raise expected exception"
 

parsl/tests/test_curvezmq.py

@@ -0,0 +1,455 @@
+import os
+import pathlib
+from typing import Union
+from unittest import mock
+
+import pytest
+import zmq
+import zmq.auth
+from zmq.auth.thread import ThreadAuthenticator
+
+from parsl import curvezmq
+
+ADDR = "tcp://127.0.0.1"
+
+
+def get_server_socket(ctx: curvezmq.ServerContext):
+    sock = ctx.socket(zmq.PULL)
+    sock.setsockopt(zmq.RCVTIMEO, 200)
+    sock.setsockopt(zmq.LINGER, 0)
+    port = sock.bind_to_random_port(ADDR)
+    return sock, port
+
+
+def get_client_socket(ctx: curvezmq.ClientContext, port: int):
+    sock = ctx.socket(zmq.PUSH)
+    sock.setsockopt(zmq.SNDTIMEO, 200)
+    sock.setsockopt(zmq.LINGER, 0)
+    sock.connect(f"{ADDR}:{port}")
+    return sock
+
+
+def get_external_server_socket(
+    ctx: Union[curvezmq.ServerContext, zmq.Context], secret_key: bytes
+):
+    sock = ctx.socket(zmq.PULL)
+    sock.setsockopt(zmq.RCVTIMEO, 200)
+    sock.setsockopt(zmq.LINGER, 0)
+    sock.setsockopt(zmq.CURVE_SECRETKEY, secret_key)
+    sock.setsockopt(zmq.CURVE_SERVER, True)
+    port = sock.bind_to_random_port(ADDR)
+    return sock, port
+
+
+def get_external_client_socket(
+    ctx: Union[curvezmq.ClientContext, zmq.Context],
+    public_key: bytes,
+    secret_key: bytes,
+    server_key: bytes,
+    port: int,
+):
+    sock = ctx.socket(zmq.PUSH)
+    sock.setsockopt(zmq.LINGER, 0)
+    sock.setsockopt(zmq.CURVE_PUBLICKEY, public_key)
+    sock.setsockopt(zmq.CURVE_SECRETKEY, secret_key)
+    sock.setsockopt(zmq.CURVE_SERVERKEY, server_key)
+    sock.connect(f"{ADDR}:{port}")
+    return sock
+
+
+@pytest.fixture
+def encrypted(request: pytest.FixtureRequest):
+    if hasattr(request, "param"):
+        return request.param
+    return True
+
+
+@pytest.fixture
+def cert_dir(encrypted: bool, tmpd_cwd: pathlib.Path):
+    if not encrypted:
+        return None
+    return curvezmq.create_certificates(tmpd_cwd)
+
+
+@pytest.fixture
+def server_ctx(cert_dir: Union[str, None]):
+    ctx = curvezmq.ServerContext(cert_dir)
+    yield ctx
+    ctx.destroy()
+
+
+@pytest.fixture
+def client_ctx(cert_dir: Union[str, None]):
+    ctx = curvezmq.ClientContext(cert_dir)
+    yield ctx
+    ctx.destroy()
+
+
+@pytest.fixture
+def zmq_ctx():
+    ctx = zmq.Context()
+    yield ctx
+    ctx.destroy()
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_client_context_init(cert_dir: Union[str, None]):
+    ctx = curvezmq.ClientContext(cert_dir=cert_dir)
+
+    assert ctx.cert_dir == cert_dir
+    if cert_dir is None:
+        assert not ctx.encrypted
+    else:
+        assert ctx.encrypted
+
+    ctx.destroy()
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_server_context_init(cert_dir: Union[str, None]):
+    ctx = curvezmq.ServerContext(cert_dir=cert_dir)
+
+    assert ctx.cert_dir == cert_dir
+    if cert_dir is None:
+        assert not ctx.encrypted
+        assert not ctx.auth_thread
+    else:
+        assert ctx.encrypted
+        assert isinstance(ctx.auth_thread, ThreadAuthenticator)
+
+    ctx.destroy()
+
+
+@pytest.mark.local
+def test_create_certificates(tmpd_cwd: pathlib.Path):
+    cert_dir = tmpd_cwd / "certificates"
+    assert not os.path.exists(cert_dir)
+
+    ret = curvezmq.create_certificates(tmpd_cwd)
+
+    assert str(cert_dir) == ret
+    assert os.path.exists(cert_dir)
+    assert os.stat(cert_dir).st_mode & 0o777 == 0o700
+    assert len(os.listdir(cert_dir)) == 4
+
+
+@pytest.mark.local
+def test_create_certificates_overwrite(tmpd_cwd: pathlib.Path):
+    cert_dir = curvezmq.create_certificates(tmpd_cwd)
+    client_pub_1, client_sec_1 = curvezmq._load_certificate(cert_dir, name="client")
+    server_pub_1, server_sec_1 = curvezmq._load_certificate(cert_dir, name="server")
+
+    curvezmq.create_certificates(tmpd_cwd)
+    client_pub_2, client_sec_2 = curvezmq._load_certificate(cert_dir, name="client")
+    server_pub_2, server_sec_2 = curvezmq._load_certificate(cert_dir, name="server")
+
+    assert client_pub_1 != client_pub_2
+    assert client_sec_1 != client_sec_2
+    assert server_pub_1 != server_pub_2
+    assert server_sec_1 != server_sec_2
+
+
+@pytest.mark.local
+def test_cert_dir_not_private(tmpd_cwd: pathlib.Path):
+    cert_dir = tmpd_cwd / "certificates"
+    os.makedirs(cert_dir, mode=0o777)
+    client_ctx = curvezmq.ClientContext(cert_dir)
+    server_ctx = curvezmq.ServerContext(cert_dir)
+
+    err_msg = "directory must be private"
+
+    with pytest.raises(OSError) as pyt_e:
+        client_ctx.socket(zmq.REQ)
+    assert err_msg in str(pyt_e.value)
+
+    with pytest.raises(OSError) as pyt_e:
+        server_ctx.socket(zmq.REP)
+    assert err_msg in str(pyt_e.value)
+
+    client_ctx.destroy()
+    server_ctx.destroy()
+
+
+@pytest.mark.local
+def test_missing_cert_dir():
+    cert_dir = "/bad/cert/dir"
+    client_ctx = curvezmq.ClientContext(cert_dir)
+    server_ctx = curvezmq.ServerContext(cert_dir)
+
+    err_msg = "No such file or directory"
+
+    with pytest.raises(FileNotFoundError) as pyt_e:
+        client_ctx.socket(zmq.REQ)
+    assert err_msg in str(pyt_e.value)
+
+    with pytest.raises(FileNotFoundError) as pyt_e:
+        server_ctx.socket(zmq.REP)
+    assert err_msg in str(pyt_e.value)
+
+    client_ctx.destroy()
+    server_ctx.destroy()
+
+
+@pytest.mark.local
+def test_missing_secret_file(tmpd_cwd: pathlib.Path):
+    cert_dir = tmpd_cwd / "certificates"
+    os.makedirs(cert_dir, mode=0o700)
+
+    client_ctx = curvezmq.ClientContext(cert_dir)
+    server_ctx = curvezmq.ServerContext(cert_dir)
+
+    err_msg = "Invalid certificate file"
+
+    with pytest.raises(OSError) as pyt_e:
+        client_ctx.socket(zmq.REQ)
+    assert err_msg in str(pyt_e.value)
+
+    with pytest.raises(OSError) as pyt_e:
+        server_ctx.socket(zmq.REP)
+    assert err_msg in str(pyt_e.value)
+
+    client_ctx.destroy()
+    server_ctx.destroy()
+
+
+@pytest.mark.local
+def test_bad_secret_file(tmpd_cwd: pathlib.Path):
+    client_sec = tmpd_cwd / "client.key_secret"
+    server_sec = tmpd_cwd / "server.key_secret"
+    client_sec.write_text("bad")
+    server_sec.write_text("boy")
+
+    client_ctx = curvezmq.ClientContext(tmpd_cwd)
+    server_ctx = curvezmq.ServerContext(tmpd_cwd)
+
+    err_msg = "No public key found"
+
+    with pytest.raises(ValueError) as pyt_e:
+        client_ctx.socket(zmq.REQ)
+    assert err_msg in str(pyt_e.value)
+
+    with pytest.raises(ValueError) as pyt_e:
+        server_ctx.socket(zmq.REP)
+    assert err_msg in str(pyt_e.value)
+
+    client_ctx.destroy()
+    server_ctx.destroy()
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_client_context_term(client_ctx: curvezmq.ClientContext):
+    assert not client_ctx.closed
+
+    client_ctx.term()
+
+    assert client_ctx.closed
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_server_context_term(server_ctx: curvezmq.ServerContext, encrypted: bool):
+    assert not server_ctx.closed
+    if encrypted:
+        assert server_ctx.auth_thread
+        assert server_ctx.auth_thread.pipe
+
+    server_ctx.term()
+
+    assert server_ctx.closed
+    if encrypted:
+        assert server_ctx.auth_thread
+        assert not server_ctx.auth_thread.pipe
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_client_context_destroy(client_ctx: curvezmq.ClientContext):
+    sock = client_ctx.socket(zmq.REP)
+
+    assert not client_ctx.closed
+
+    client_ctx.destroy()
+
+    assert sock.closed
+    assert client_ctx.closed
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_server_context_destroy(server_ctx: curvezmq.ServerContext, encrypted: bool):
+    sock = server_ctx.socket(zmq.REP)
+
+    assert not server_ctx.closed
+    if encrypted:
+        assert server_ctx.auth_thread
+        assert server_ctx.auth_thread.pipe
+
+    server_ctx.destroy()
+
+    assert sock.closed
+    assert server_ctx.closed
+    if encrypted:
+        assert server_ctx.auth_thread
+        assert not server_ctx.auth_thread.pipe
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_client_context_recreate(client_ctx: curvezmq.ClientContext):
+    hidden_ctx = client_ctx._ctx
+    sock = client_ctx.socket(zmq.REQ)
+
+    assert not sock.closed
+    assert not client_ctx.closed
+
+    client_ctx.recreate()
+
+    assert sock.closed
+    assert not client_ctx.closed
+    assert hidden_ctx != client_ctx._ctx
+    assert hidden_ctx.closed
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_server_context_recreate(server_ctx: curvezmq.ServerContext, encrypted: bool):
+    hidden_ctx = server_ctx._ctx
+    sock = server_ctx.socket(zmq.REP)
+
+    assert not sock.closed
+    assert not server_ctx.closed
+    if encrypted:
+        assert server_ctx.auth_thread
+        auth_thread = server_ctx.auth_thread
+        assert auth_thread.pipe
+
+    server_ctx.recreate()
+
+    assert sock.closed
+    assert not server_ctx.closed
+    assert hidden_ctx.closed
+    assert hidden_ctx != server_ctx._ctx
+    if encrypted:
+        assert server_ctx.auth_thread
+        assert auth_thread != server_ctx.auth_thread
+        assert server_ctx.auth_thread.pipe
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("encrypted", (True, False), indirect=True)
+def test_connection(
+    server_ctx: curvezmq.ServerContext, client_ctx: curvezmq.ClientContext
+):
+    server_socket, port = get_server_socket(server_ctx)
+    client_socket = get_client_socket(client_ctx, port)
+
+    msg = b"howdy"
+    client_socket.send(msg)
+    recv = server_socket.recv()
+
+    assert recv == msg
+
+
+@pytest.mark.local
+@mock.patch.object(curvezmq, "_load_certificate")
+def test_invalid_key_format(
+    mock_load_cert: mock.MagicMock,
+    server_ctx: curvezmq.ServerContext,
+    client_ctx: curvezmq.ClientContext,
+):
+    mock_load_cert.return_value = (b"badkey", b"badkey")
+
+    with pytest.raises(ValueError) as e1_info:
+        server_ctx.socket(zmq.REP)
+    with pytest.raises(ValueError) as e2_info:
+        client_ctx.socket(zmq.REQ)
+    e1, e2 = e1_info.exconly, e2_info.exconly
+
+    assert str(e1) == str(e2)
+    assert "Invalid CurveZMQ key format" in str(e1)
+
+
+@pytest.mark.local
+def test_invalid_client_keys(server_ctx: curvezmq.ServerContext, zmq_ctx: zmq.Context):
+    server_socket, port = get_server_socket(server_ctx)
+
+    cert_dir = server_ctx.cert_dir
+    assert cert_dir  # For mypy
+    public_key, secret_key = curvezmq._load_certificate(cert_dir, "client")
+    server_key, _ = curvezmq._load_certificate(cert_dir, "server")
+
+    BAD_PUB_KEY, BAD_SEC_KEY = zmq.curve_keypair()
+    msg = b"howdy"
+
+    client_socket = get_external_client_socket(
+        zmq_ctx,
+        public_key,
+        secret_key,
+        server_key,
+        port,
+    )
+    client_socket.send(msg)
+    assert server_socket.recv() == msg
+
+    client_socket = get_external_client_socket(
+        zmq_ctx,
+        BAD_PUB_KEY,
+        BAD_SEC_KEY,
+        server_key,
+        port,
+    )
+    client_socket.send(msg)
+    with pytest.raises(zmq.Again):
+        server_socket.recv()
+
+    client_socket = get_external_client_socket(
+        zmq_ctx,
+        public_key,
+        secret_key,
+        BAD_PUB_KEY,
+        port,
+    )
+    client_socket.send(msg)
+    with pytest.raises(zmq.Again):
+        server_socket.recv()
+
+    # Ensure sockets are operational
+    client_socket = get_external_client_socket(
+        zmq_ctx,
+        public_key,
+        secret_key,
+        server_key,
+        port,
+    )
+    client_socket.send(msg)
+    assert server_socket.recv() == msg
+
+
+@pytest.mark.local
+def test_invalid_server_key(client_ctx: curvezmq.ClientContext, zmq_ctx: zmq.Context):
+    cert_dir = client_ctx.cert_dir
+    assert cert_dir  # For mypy
+    _, secret_key = curvezmq._load_certificate(cert_dir, "server")
+
+    _, BAD_SEC_KEY = zmq.curve_keypair()
+    msg = b"howdy"
+
+    server_socket, port = get_external_server_socket(zmq_ctx, secret_key)
+    client_socket = get_client_socket(client_ctx, port)
+    client_socket.send(msg)
+    assert server_socket.recv() == msg
+
+    server_socket, port = get_external_server_socket(zmq_ctx, BAD_SEC_KEY)
+    client_socket = get_client_socket(client_ctx, port)
+    client_socket.send(msg)
+    with pytest.raises(zmq.Again):
+        server_socket.recv()
+
+    # Ensure sockets are operational
+    server_socket, port = get_external_server_socket(zmq_ctx, secret_key)
+    client_socket = get_client_socket(client_ctx, port)
+    client_socket.send(msg)
+    assert server_socket.recv() == msg

parsl/tests/test_data/test_file_apps.py

@@ -35,10 +35,10 @@ def increment(inputs=(), outputs=(), stdout=None, stderr=None):
 
 
 @pytest.mark.staging_required
-def test_increment(tmp_path, depth=5):
+def test_increment(tmpd_cwd, depth=5):
     """Test simple pipeline A->B...->N"""
     # Test setup
-    first_fpath = tmp_path / "test0.txt"
+    first_fpath = tmpd_cwd / "test0.txt"
     first_fpath.write_text("0\n")
 
     prev = [File(first_fpath)]
@@ -46,9 +46,9 @@ def test_increment(tmp_path, depth=5):
     for i in range(1, depth):
         f = increment(
             inputs=prev,
-            outputs=[File(tmp_path / f"test{i}.txt")],
-            stdout=tmp_path / f"incr{i}.out",
-            stderr=tmp_path / f"incr{i}.err",
+            outputs=[File(tmpd_cwd / f"test{i}.txt")],
+            stdout=tmpd_cwd / f"incr{i}.out",
+            stderr=tmpd_cwd / f"incr{i}.err",
         )
         prev = f.outputs
         futs.append((i, prev[0]))

parsl/tests/test_data/test_file_staging.py

@@ -11,10 +11,10 @@ def cat(inputs=(), outputs=(), stdout=None, stderr=None):
 
 
 @pytest.mark.staging_required
-def test_regression_200(tmp_path):
+def test_regression_200(tmpd_cwd):
     """Regression test for #200. Pickleablility of Files"""
-    opath = tmp_path / "test_output.txt"
-    fpath = tmp_path / "test.txt"
+    opath = tmpd_cwd / "test_output.txt"
+    fpath = tmpd_cwd / "test.txt"
 
     fpath.write_text("Hello World")
     f = cat(inputs=[File(fpath)], outputs=[File(opath)])

parsl/tests/test_docs/test_kwargs.py

@@ -19,7 +19,7 @@ def test_inputs():
     assert reduce_future.result() == 6
 
 
-def test_outputs(tmpdir):
+def test_outputs(tmpd_cwd):
     @python_app()
     def write_app(message, outputs=()):
         """Write a single message to every file in outputs"""
@@ -28,8 +28,8 @@ def test_outputs(tmpdir):
                 print(message, file=fp)
 
     to_write = [
-        File(Path(tmpdir) / 'output-0.txt'),
-        File(Path(tmpdir) / 'output-1.txt')
+        File(Path(tmpd_cwd) / 'output-0.txt'),
+        File(Path(tmpd_cwd) / 'output-1.txt')
     ]
     write_app('Hello!', outputs=to_write).result()
     for path in to_write: