parsl 2024.3.18__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/tests/test_mpi_apps/test_resource_spec.py

@@ -1,41 +1,36 @@
 import contextlib
 import logging
 import os
+import queue
 import typing
-
+import unittest
+from typing import Dict
+from unittest import mock
 
 import pytest
-import unittest
 
-import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.htex_local import fresh_config
-from typing import Dict
+from parsl.executors.errors import InvalidResourceSpecification
+from parsl.executors.high_throughput.executor import HighThroughputExecutor
+from parsl.executors.high_throughput.mpi_executor import MPIExecutor
 from parsl.executors.high_throughput.mpi_resource_management import (
+    get_nodes_in_batchjob,
     get_pbs_hosts_list,
     get_slurm_hosts_list,
-    get_nodes_in_batchjob,
     identify_scheduler,
 )
-from parsl.executors.high_throughput.mpi_prefix_composer import (
-    validate_resource_spec,
-    InvalidResourceSpecification
-)
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+from parsl.tests.configs.htex_local import fresh_config
 
 EXECUTOR_LABEL = "MPI_TEST"
 
 
-def local_setup():
+def local_config():
     config = fresh_config()
     config.executors[0].label = EXECUTOR_LABEL
     config.executors[0].max_workers_per_node = 1
-    parsl.load(config)
-
-
-def local_teardown():
-    logging.warning("Exiting")
-    parsl.dfk().cleanup()
-    parsl.clear()
+    return config
 
 
 @python_app
@@ -54,23 +49,6 @@ def get_env_vars(parsl_resource_specification: Dict = {}) -> Dict:
     return parsl_vars
 
 
-@pytest.mark.local
-def test_resource_spec_env_vars():
-    resource_spec = {
-        "num_nodes": 4,
-        "ranks_per_node": 2,
-    }
-
-    assert double(5).result() == 10
-
-    future = get_env_vars(parsl_resource_specification=resource_spec)
-
-    result = future.result()
-    assert isinstance(result, Dict)
-    assert result["PARSL_NUM_NODES"] == str(resource_spec["num_nodes"])
-    assert result["PARSL_RANKS_PER_NODE"] == str(resource_spec["ranks_per_node"])
-
-
 @pytest.mark.local
 @unittest.mock.patch("subprocess.check_output", return_value=b"c203-031\nc203-032\n")
 def test_slurm_mocked_mpi_fetch(subprocess_check):
@@ -89,16 +67,6 @@ def add_to_path(path: os.PathLike) -> typing.Generator[None, None, None]:
         os.environ["PATH"] = old_path
 
 
-@pytest.mark.local
-@pytest.mark.skip
-def test_slurm_mpi_fetch():
-    logging.warning(f"Current pwd : {os.path.dirname(__file__)}")
-    with add_to_path(os.path.dirname(__file__)):
-        logging.warning(f"PATH: {os.environ['PATH']}")
-        nodeinfo = get_slurm_hosts_list()
-    logging.warning(f"Got : {nodeinfo}")
-
-
 @contextlib.contextmanager
 def mock_pbs_nodefile(nodefile: str = "pbs_nodefile") -> typing.Generator[None, None, None]:
     cwd = os.path.abspath(os.path.dirname(__file__))
@@ -130,16 +98,41 @@ def test_top_level():
 @pytest.mark.parametrize(
     "resource_spec, exception",
     (
+
         ({"num_nodes": 2, "ranks_per_node": 1}, None),
         ({"launcher_options": "--debug_foo"}, None),
         ({"num_nodes": 2, "BAD_OPT": 1}, InvalidResourceSpecification),
-        ({}, None),
+        ({}, InvalidResourceSpecification),
     )
 )
-def test_resource_spec(resource_spec: Dict, exception):
+def test_mpi_resource_spec(resource_spec: Dict, exception):
+    """Test validation of resource_specification in MPIExecutor"""
+
+    mpi_ex = MPIExecutor(provider=LocalProvider(launcher=SimpleLauncher()))
+    mpi_ex.outgoing_q = mock.Mock(spec=queue.Queue)
+
     if exception:
         with pytest.raises(exception):
-            validate_resource_spec(resource_spec)
+            mpi_ex.validate_resource_spec(resource_spec)
     else:
-        result = validate_resource_spec(resource_spec)
+        result = mpi_ex.validate_resource_spec(resource_spec)
         assert result is None
+
+
+@pytest.mark.local
+@pytest.mark.parametrize(
+    "resource_spec",
+    (
+        {"num_nodes": 2, "ranks_per_node": 1},
+        {"launcher_options": "--debug_foo"},
+        {"BAD_OPT": 1},
+    )
+)
+def test_mpi_resource_spec_passed_to_htex(resource_spec: dict):
+    """HTEX should reject every resource_spec"""
+
+    htex = HighThroughputExecutor()
+    htex.outgoing_q = mock.Mock(spec=queue.Queue)
+
+    with pytest.raises(InvalidResourceSpecification):
+        htex.validate_resource_spec(resource_spec)

parsl/tests/test_providers/test_kubernetes_provider.py

@@ -0,0 +1,102 @@
+import re
+from unittest import mock
+
+import pytest
+
+from parsl.providers.kubernetes.kube import KubernetesProvider
+from parsl.tests.test_utils.test_sanitize_dns import DNS_SUBDOMAIN_REGEX
+
+_MOCK_BASE = "parsl.providers.kubernetes.kube"
+
+
+@pytest.fixture(autouse=True)
+def mock_kube_config():
+    with mock.patch(f"{_MOCK_BASE}.config") as mock_config:
+        mock_config.load_kube_config.return_value = None
+        yield mock_config
+
+
+@pytest.fixture
+def mock_kube_client():
+    mock_client = mock.MagicMock()
+    with mock.patch(f"{_MOCK_BASE}.client.CoreV1Api") as mock_api:
+        mock_api.return_value = mock_client
+        yield mock_client
+
+
+@pytest.mark.local
+def test_submit_happy_path(mock_kube_client: mock.MagicMock):
+    image = "test-image"
+    namespace = "test-namespace"
+    cmd_string = "test-command"
+    volumes = [("test-volume", "test-mount-path")]
+    service_account_name = "test-service-account"
+    annotations = {"test-annotation": "test-value"}
+    max_cpu = 2
+    max_mem = "2Gi"
+    init_cpu = 1
+    init_mem = "1Gi"
+    provider = KubernetesProvider(
+        image=image,
+        persistent_volumes=volumes,
+        namespace=namespace,
+        service_account_name=service_account_name,
+        annotations=annotations,
+        max_cpu=max_cpu,
+        max_mem=max_mem,
+        init_cpu=init_cpu,
+        init_mem=init_mem,
+    )
+
+    job_name = "test.job.name"
+    job_id = provider.submit(cmd_string=cmd_string, tasks_per_node=1, job_name=job_name)
+
+    assert job_id in provider.resources
+    assert mock_kube_client.create_namespaced_pod.call_count == 1
+
+    call_args = mock_kube_client.create_namespaced_pod.call_args[1]
+    pod = call_args["body"]
+    container = pod.spec.containers[0]
+    volume = container.volume_mounts[0]
+
+    assert image == container.image
+    assert namespace == call_args["namespace"]
+    assert any(cmd_string in arg for arg in container.args)
+    assert volumes[0] == (volume.name, volume.mount_path)
+    assert service_account_name == pod.spec.service_account_name
+    assert annotations == pod.metadata.annotations
+    assert str(max_cpu) == container.resources.limits["cpu"]
+    assert max_mem == container.resources.limits["memory"]
+    assert str(init_cpu) == container.resources.requests["cpu"]
+    assert init_mem == container.resources.requests["memory"]
+    assert job_id == pod.metadata.labels["parsl-job-id"]
+    assert job_id == container.name
+    assert f"{job_name}.{job_id}" == pod.metadata.name
+
+
+@pytest.mark.local
+@mock.patch(f"{_MOCK_BASE}.KubernetesProvider._create_pod")
+@pytest.mark.parametrize("char", (".", "-"))
+def test_submit_pod_name_includes_job_id(mock_create_pod: mock.MagicMock, char: str):
+    provider = KubernetesProvider(image="test-image")
+
+    job_name = "a." * 121 + f"a{char}" + "a" * 9
+    assert len(job_name) == 253  # Max length for pod name
+    job_id = provider.submit(cmd_string="test-command", tasks_per_node=1, job_name=job_name)
+
+    expected_pod_name = job_name[:253 - len(job_id) - 2] + f".{job_id}"
+    actual_pod_name = mock_create_pod.call_args[1]["pod_name"]
+    assert re.match(DNS_SUBDOMAIN_REGEX, actual_pod_name)
+    assert expected_pod_name == actual_pod_name
+
+
+@pytest.mark.local
+@mock.patch(f"{_MOCK_BASE}.KubernetesProvider._create_pod")
+@mock.patch(f"{_MOCK_BASE}.logger")
+@pytest.mark.parametrize("job_name", ("", ".", "-", "a.-.a", "$$$"))
+def test_submit_invalid_job_name(mock_logger: mock.MagicMock, mock_create_pod: mock.MagicMock, job_name: str):
+    provider = KubernetesProvider(image="test-image")
+    job_id = provider.submit(cmd_string="test-command", tasks_per_node=1, job_name=job_name)
+    assert mock_logger.warning.call_count == 1
+    assert f"Invalid pod name '{job_name}' for job '{job_id}'" in mock_logger.warning.call_args[0][0]
+    assert f"parsl.kube.{job_id}" == mock_create_pod.call_args[1]["pod_name"]

parsl/tests/test_providers/test_local_provider.py

@@ -1,7 +1,6 @@
 import logging
 import os
 import pathlib
-import pytest
 import random
 import shutil
 import socket
@@ -10,7 +9,8 @@ import tempfile
 import threading
 import time
 
-from parsl.channels import LocalChannel, SSHChannel
+import pytest
+
 from parsl.jobs.states import JobState
 from parsl.launchers import SingleNodeLauncher
 from parsl.providers import LocalProvider
@@ -62,140 +62,11 @@ def _run_tests(p: LocalProvider):
 def test_local_channel():
     with tempfile.TemporaryDirectory() as script_dir:
         script_dir = tempfile.mkdtemp()
-        p = LocalProvider(channel=LocalChannel(), launcher=SingleNodeLauncher(debug=False))
+        p = LocalProvider(launcher=SingleNodeLauncher(debug=False))
         p.script_dir = script_dir
         _run_tests(p)
 
 
-SSHD_CONFIG = """
-Port {port}
-ListenAddress 127.0.0.1
-HostKey {hostkey}
-AuthorizedKeysFile {connpubkey}
-AuthenticationMethods publickey
-StrictModes no
-Subsystem sftp {sftp_path}
-"""
-
-
-# It would probably be better, when more formalized site testing comes into existence, to
-# use a site-testing provided server/configuration instead of the current scheme
-@pytest.mark.local
-@pytest.mark.sshd_required
-def test_ssh_channel():
-    with tempfile.TemporaryDirectory() as config_dir:
-        sshd_thread, priv_key, server_port = _start_sshd(config_dir)
-        try:
-            with tempfile.TemporaryDirectory() as remote_script_dir:
-                # The SSH library fails to add the new host key to the file if the file does not
-                # already exist, so create it here.
-                pathlib.Path('{}/known.hosts'.format(config_dir)).touch(mode=0o600)
-                script_dir = tempfile.mkdtemp()
-                p = LocalProvider(channel=SSHChannel('127.0.0.1', port=server_port,
-                                                     script_dir=remote_script_dir,
-                                                     host_keys_filename='{}/known.hosts'.format(config_dir),
-                                                     key_filename=priv_key),
-                                  launcher=SingleNodeLauncher(debug=False))
-                p.script_dir = script_dir
-                _run_tests(p)
-        finally:
-            _stop_sshd(sshd_thread)
-
-
-def _stop_sshd(sshd_thread):
-    sshd_thread.stop()
-
-
-class SSHDThread(threading.Thread):
-    def __init__(self, config_file):
-        threading.Thread.__init__(self, daemon=True)
-        self.config_file = config_file
-        self.stop_flag = False
-        self.error = None
-
-    def run(self):
-        try:
-            # sshd needs to be run with an absolute path, hence the call to which()
-            sshpath = shutil.which('sshd')
-            assert sshpath is not None, "can find sshd executable"
-            p = subprocess.Popen([sshpath, '-D', '-f', self.config_file],
-                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            while True:
-                ec = p.poll()
-                if self.stop_flag:
-                    p.terminate()
-                    break
-                elif ec is None:
-                    time.sleep(0.1)
-                elif ec == 0:
-                    self.error = Exception('sshd exited prematurely: {}{}'.format(p.stdout.read(),
-                                                                                  p.stderr.read()))
-                    break
-                else:
-                    self.error = Exception('sshd failed: {}{}'.format(p.stdout.read(),
-                                                                      p.stderr.read()))
-                    break
-        except Exception as ex:
-            logger.exception("SSHDThread exception from run loop")
-            self.error = ex
-
-    def stop(self):
-        self.stop_flag = True
-
-
-def _start_sshd(config_dir: str):
-    server_config, priv_key, port = _init_sshd(config_dir)
-    sshd_thread = SSHDThread(server_config)
-    sshd_thread.start()
-    time.sleep(1.0)
-    if not sshd_thread.is_alive():
-        raise Exception('Failed to start sshd: {}'.format(sshd_thread.error))
-    return sshd_thread, priv_key, port
-
-
-def _init_sshd(config_dir):
-    hostkey = '{}/hostkey'.format(config_dir)
-    connkey = '{}/connkey'.format(config_dir)
-    os.system('ssh-keygen -b 2048 -t rsa -q -N "" -f {}'.format(hostkey))
-    os.system('ssh-keygen -b 2048 -t rsa -q -N "" -f {}'.format(connkey))
-    port = _find_free_port(22222)
-    server_config_str = SSHD_CONFIG.format(port=port, hostkey=hostkey,
-                                           connpubkey='{}.pub'.format(connkey),
-                                           sftp_path=_get_system_sftp_path())
-    server_config = '{}/sshd_config'.format(config_dir)
-    with open(server_config, 'w') as f:
-        f.write(server_config_str)
-    return server_config, connkey, port
-
-
-def _get_system_sftp_path():
-    try:
-        with open('/etc/ssh/sshd_config') as f:
-            line = f.readline()
-            while line:
-                tokens = line.split()
-                if tokens[0] == 'Subsystem' and tokens[1] == 'sftp':
-                    return tokens[2]
-                line = f.readline()
-    except Exception:
-        pass
-    return '/usr/lib/openssh/sftp-server'
-
-
-def _find_free_port(start: int):
-    port = start
-    while port < 65535:
-        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        try:
-            s.bind(('127.0.0.1', port))
-            s.close()
-            return port
-        except Exception:
-            pass
-        port += random.randint(1, 20)
-    raise Exception('Could not find free port')
-
-
 def _run(p: LocalProvider, command: str, np: int = 1):
     id = p.submit(command, np)
     return _wait(p, id)

parsl/tests/test_providers/test_pbspro_template.py

@@ -1,9 +1,8 @@
 import random
-
 from unittest import mock
+
 import pytest
 
-from parsl.channels import LocalChannel
 from parsl.providers import PBSProProvider
 
 
@@ -12,7 +11,7 @@ def test_submit_script_basic(tmp_path):
     """Test slurm resources table"""
 
     provider = PBSProProvider(
-        queue="debug", channel=LocalChannel(script_dir=tmp_path)
+        queue="debug"
     )
     provider.script_dir = tmp_path
     job_id = str(random.randint(55000, 59000))

parsl/tests/test_providers/test_slurm_template.py

@@ -1,10 +1,9 @@
 import logging
 import random
-
 from unittest import mock
+
 import pytest
 
-from parsl.channels import LocalChannel
 from parsl.providers import SlurmProvider
 
 
@@ -13,7 +12,7 @@ def test_submit_script_basic(tmp_path):
     """Test slurm resources table"""
 
     provider = SlurmProvider(
-        partition="debug", channel=LocalChannel(script_dir=tmp_path)
+        partition="debug"
     )
     provider.script_dir = tmp_path
     job_id = str(random.randint(55000, 59000))

parsl/tests/test_providers/test_submiterror_deprecation.py

@@ -1,7 +1,8 @@
-import pytest
 import random
 import string
 
+import pytest
+
 from parsl.providers.errors import SubmitException
 
 

parsl/tests/test_python_apps/test_context_manager.py

@@ -0,0 +1,128 @@
+from concurrent.futures import Future
+from threading import Event
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.dataflow.dflow import DataFlowKernel, DataFlowKernelLoader
+from parsl.errors import NoDataFlowKernelError
+from parsl.tests.configs.local_threads import fresh_config
+
+
+@parsl.python_app
+def square(x):
+    return x * x
+
+
+@parsl.bash_app
+def foo(x, stdout='foo.stdout'):
+    return f"echo {x + 1}"
+
+
+@parsl.python_app
+def wait_for_event(ev: Event):
+    ev.wait()
+
+
+@parsl.python_app
+def raise_app():
+    raise RuntimeError("raise_app deliberate failure")
+
+
+@pytest.mark.local
+def test_within_context_manger(tmpd_cwd):
+    config = fresh_config()
+    with parsl.load(config=config) as dfk:
+        assert isinstance(dfk, DataFlowKernel)
+
+        bash_future = foo(1, stdout=tmpd_cwd / 'foo.stdout')
+        assert bash_future.result() == 0
+
+        with open(tmpd_cwd / 'foo.stdout', 'r') as f:
+            assert f.read() == "2\n"
+
+    with pytest.raises(NoDataFlowKernelError) as excinfo:
+        square(2).result()
+    assert str(excinfo.value) == "Must first load config"
+
+
+@pytest.mark.local
+def test_exit_skip():
+    config = fresh_config()
+    config.exit_mode = "skip"
+
+    with parsl.load(config) as dfk:
+        ev = Event()
+        fut = wait_for_event(ev)
+        # deliberately don't wait for this to finish, so that the context
+        # manager can exit
+
+    assert parsl.dfk() is dfk, "global dfk should be left in place by skip mode"
+
+    assert not fut.done(), "wait_for_event should not be done yet"
+    ev.set()
+
+    # now we can wait for that result...
+    fut.result()
+    assert fut.done(), "wait_for_event should complete outside of context manager in 'skip' mode"
+
+    # now cleanup the DFK that the above `with` block
+    # deliberately avoided doing...
+    dfk.cleanup()
+
+
+# 'wait' mode has two cases to test:
+# 1. that we wait when there is no exception
+# 2. that we do not wait when there is an exception
+@pytest.mark.local
+def test_exit_wait_no_exception():
+    config = fresh_config()
+    config.exit_mode = "wait"
+
+    with parsl.load(config) as dfk:
+        fut = square(1)
+        # deliberately don't wait for this to finish, so that the context
+        # manager can exit
+
+    assert fut.done(), "This future should be marked as done before the context manager exits"
+
+    assert dfk.cleanup_called, "The DFK should have been cleaned up by the context manager"
+    assert DataFlowKernelLoader._dfk is None, "The global DFK should have been removed"
+
+
+@pytest.mark.local
+def test_exit_wait_exception():
+    config = fresh_config()
+    config.exit_mode = "wait"
+
+    with pytest.raises(RuntimeError):
+        with parsl.load(config) as dfk:
+            # we'll never fire this future
+            fut_never = Future()
+
+            fut_raise = raise_app()
+
+            fut_depend = square(fut_never)
+
+            # this should cause an exception, which should cause the context
+            # manager to exit, without waiting for fut_depend to finish.
+            fut_raise.result()
+
+    assert dfk.cleanup_called, "The DFK should have been cleaned up by the context manager"
+    assert DataFlowKernelLoader._dfk is None, "The global DFK should have been removed"
+    assert fut_raise.exception() is not None, "fut_raise should contain an exception"
+    assert not fut_depend.done(), "fut_depend should have been left un-done (due to dependency failure)"
+
+
+@pytest.mark.local
+def test_exit_wrong_mode():
+
+    with pytest.raises(Exception) as ex:
+        Config(exit_mode="wrongmode")
+
+    # with typeguard 4.x this is TypeCheckError,
+    # with typeguard 2.x this is TypeError
+    # we can't instantiate TypeCheckError if we're in typeguard 2.x environment
+    # because it does not exist... so check name using strings.
+    assert ex.type.__name__ == "TypeCheckError" or ex.type.__name__ == "TypeError"

parsl/tests/test_python_apps/test_dep_standard_futures.py

@@ -1,6 +1,7 @@
+from concurrent.futures import Future
+
 import parsl
 from parsl.dataflow.errors import DependencyError
-from concurrent.futures import Future
 
 
 @parsl.python_app

parsl/tests/test_python_apps/test_dependencies_deep.py

@@ -0,0 +1,59 @@
+import inspect
+from concurrent.futures import Future
+from typing import Any, Callable, Dict
+
+import pytest
+
+import parsl
+from parsl.executors.base import ParslExecutor
+
+# N is the number of tasks to chain
+# With mid-2024 Parsl, N>140 causes Parsl to hang
+N = 100
+
+# MAX_STACK is the maximum Python stack depth allowed for either
+# task submission to an executor or execution of a task.
+# With mid-2024 Parsl, 2-3 stack entries will be used per
+# recursively launched parsl task. So this should be smaller than
+# 2*N, but big enough to allow regular pytest+parsl stuff to
+# happen.
+MAX_STACK = 50
+
+
+def local_config():
+    return parsl.Config(executors=[ImmediateExecutor()])
+
+
+class ImmediateExecutor(ParslExecutor):
+    def start(self):
+        pass
+
+    def shutdown(self):
+        pass
+
+    def submit(self, func: Callable, resource_specification: Dict[str, Any], *args: Any, **kwargs: Any) -> Future:
+        stack_depth = len(inspect.stack())
+        assert stack_depth < MAX_STACK, "tasks should not be launched deep in the Python stack"
+        fut: Future[None] = Future()
+        res = func(*args, **kwargs)
+        fut.set_result(res)
+        return fut
+
+
+@parsl.python_app
+def chain(upstream):
+    stack_depth = len(inspect.stack())
+    assert stack_depth < MAX_STACK, "chained dependencies should not be launched deep in the Python stack"
+
+
+@pytest.mark.local
+def test_deep_dependency_stack_depth():
+
+    fut = Future()
+    here = fut
+
+    for _ in range(N):
+        here = chain(here)
+
+    fut.set_result(None)
+    here.result()

parsl/tests/test_python_apps/test_fail.py

@@ -41,28 +41,3 @@ def test_fail_sequence(fail_probs):
 
     with pytest.raises(DependencyError):
         t_final.result()
-
-
-def test_deps(width=3):
-    """Random failures in branches of Map -> Map -> reduce"""
-    # App1   App2  ... AppN
-    futs = [random_fail(fail_prob=0.4) for _ in range(width)]
-
-    # App1   App2  ... AppN
-    # |       |        |
-    # V       V        V
-    # App1   App2  ... AppN
-
-    futs = [random_fail(fail_prob=0.8, inputs=[f]) for f in futs]
-
-    # App1   App2  ... AppN
-    #   |       |        |
-    #   V       V        V
-    # App1   App2  ... AppN
-    #    \      |       /
-    #     \     |      /
-    # App_Final
-    try:
-        random_fail(fail_prob=0, inputs=futs).result()
-    except DependencyError:
-        pass

parsl/tests/test_python_apps/test_futures.py

@@ -11,9 +11,10 @@ Same applies to datafutures, and we need to know the behavior wrt.
 2. done() called on 1, vs 2
 
 """
-import pytest
 from os.path import basename
 
+import pytest
+
 from parsl.app.app import python_app
 from parsl.data_provider.files import File