parsl 2024.5.27__py3-none-any.whl → 2024.6.10__py3-none-any.whl

parsl/tests/test_mpi_apps/test_resource_spec.py

@@ -2,26 +2,25 @@ import contextlib
 import logging
 import os
 import typing
-
+import unittest
+from typing import Dict
 
 import pytest
-import unittest
 
 import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.htex_local import fresh_config
-from typing import Dict
+from parsl.executors.high_throughput.mpi_prefix_composer import (
+    InvalidResourceSpecification,
+    MissingResourceSpecification,
+    validate_resource_spec,
+)
 from parsl.executors.high_throughput.mpi_resource_management import (
+    get_nodes_in_batchjob,
     get_pbs_hosts_list,
     get_slurm_hosts_list,
-    get_nodes_in_batchjob,
     identify_scheduler,
 )
-from parsl.executors.high_throughput.mpi_prefix_composer import (
-    validate_resource_spec,
-    InvalidResourceSpecification,
-    MissingResourceSpecification
-)
+from parsl.tests.configs.htex_local import fresh_config
 
 EXECUTOR_LABEL = "MPI_TEST"
 

parsl/tests/test_providers/test_cobalt_deprecation_warning.py

@@ -1,5 +1,7 @@
 import warnings
+
 import pytest
+
 from parsl.providers import CobaltProvider
 
 

parsl/tests/test_providers/test_local_provider.py

@@ -1,7 +1,6 @@
 import logging
 import os
 import pathlib
-import pytest
 import random
 import shutil
 import socket
@@ -10,6 +9,8 @@ import tempfile
 import threading
 import time
 
+import pytest
+
 from parsl.channels import LocalChannel, SSHChannel
 from parsl.jobs.states import JobState
 from parsl.launchers import SingleNodeLauncher

parsl/tests/test_providers/test_pbspro_template.py

@@ -1,6 +1,6 @@
 import random
-
 from unittest import mock
+
 import pytest
 
 from parsl.channels import LocalChannel

parsl/tests/test_providers/test_slurm_template.py

@@ -1,7 +1,7 @@
 import logging
 import random
-
 from unittest import mock
+
 import pytest
 
 from parsl.channels import LocalChannel

parsl/tests/test_providers/test_submiterror_deprecation.py

@@ -1,7 +1,8 @@
-import pytest
 import random
 import string
 
+import pytest
+
 from parsl.providers.errors import SubmitException
 
 

parsl/tests/test_python_apps/test_context_manager.py

@@ -1,8 +1,13 @@
-import parsl
-from parsl.tests.configs.local_threads import fresh_config
+from concurrent.futures import Future
+from threading import Event
+
 import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.dataflow.dflow import DataFlowKernel, DataFlowKernelLoader
 from parsl.errors import NoDataFlowKernelError
-from parsl.dataflow.dflow import DataFlowKernel
+from parsl.tests.configs.local_threads import fresh_config
 
 
 @parsl.python_app
@@ -15,6 +20,16 @@ def foo(x, stdout='foo.stdout'):
     return f"echo {x + 1}"
 
 
+@parsl.python_app
+def wait_for_event(ev: Event):
+    ev.wait()
+
+
+@parsl.python_app
+def raise_app():
+    raise RuntimeError("raise_app deliberate failure")
+
+
 @pytest.mark.local
 def test_within_context_manger(tmpd_cwd):
     config = fresh_config()
@@ -30,3 +45,84 @@ def test_within_context_manger(tmpd_cwd):
     with pytest.raises(NoDataFlowKernelError) as excinfo:
         square(2).result()
     assert str(excinfo.value) == "Must first load config"
+
+
+@pytest.mark.local
+def test_exit_skip():
+    config = fresh_config()
+    config.exit_mode = "skip"
+
+    with parsl.load(config) as dfk:
+        ev = Event()
+        fut = wait_for_event(ev)
+        # deliberately don't wait for this to finish, so that the context
+        # manager can exit
+
+    assert parsl.dfk() is dfk, "global dfk should be left in place by skip mode"
+
+    assert not fut.done(), "wait_for_event should not be done yet"
+    ev.set()
+
+    # now we can wait for that result...
+    fut.result()
+    assert fut.done(), "wait_for_event should complete outside of context manager in 'skip' mode"
+
+    # now cleanup the DFK that the above `with` block
+    # deliberately avoided doing...
+    dfk.cleanup()
+
+
+# 'wait' mode has two cases to test:
+# 1. that we wait when there is no exception
+# 2. that we do not wait when there is an exception
+@pytest.mark.local
+def test_exit_wait_no_exception():
+    config = fresh_config()
+    config.exit_mode = "wait"
+
+    with parsl.load(config) as dfk:
+        fut = square(1)
+        # deliberately don't wait for this to finish, so that the context
+        # manager can exit
+
+    assert fut.done(), "This future should be marked as done before the context manager exits"
+
+    assert dfk.cleanup_called, "The DFK should have been cleaned up by the context manager"
+    assert DataFlowKernelLoader._dfk is None, "The global DFK should have been removed"
+
+
+@pytest.mark.local
+def test_exit_wait_exception():
+    config = fresh_config()
+    config.exit_mode = "wait"
+
+    with pytest.raises(RuntimeError):
+        with parsl.load(config) as dfk:
+            # we'll never fire this future
+            fut_never = Future()
+
+            fut_raise = raise_app()
+
+            fut_depend = square(fut_never)
+
+            # this should cause an exception, which should cause the context
+            # manager to exit, without waiting for fut_depend to finish.
+            fut_raise.result()
+
+    assert dfk.cleanup_called, "The DFK should have been cleaned up by the context manager"
+    assert DataFlowKernelLoader._dfk is None, "The global DFK should have been removed"
+    assert fut_raise.exception() is not None, "fut_raise should contain an exception"
+    assert not fut_depend.done(), "fut_depend should have been left un-done (due to dependency failure)"
+
+
+@pytest.mark.local
+def test_exit_wrong_mode():
+
+    with pytest.raises(Exception) as ex:
+        Config(exit_mode="wrongmode")
+
+    # with typeguard 4.x this is TypeCheckError,
+    # with typeguard 2.x this is TypeError
+    # we can't instantiate TypeCheckError if we're in typeguard 2.x environment
+    # because it does not exist... so check name using strings.
+    assert ex.type.__name__ == "TypeCheckError" or ex.type.__name__ == "TypeError"

parsl/tests/test_python_apps/test_dep_standard_futures.py

@@ -1,6 +1,7 @@
+from concurrent.futures import Future
+
 import parsl
 from parsl.dataflow.errors import DependencyError
-from concurrent.futures import Future
 
 
 @parsl.python_app

parsl/tests/test_python_apps/test_dependencies_deep.py

@@ -0,0 +1,59 @@
+import inspect
+from concurrent.futures import Future
+from typing import Any, Callable, Dict
+
+import pytest
+
+import parsl
+from parsl.executors.base import ParslExecutor
+
+# N is the number of tasks to chain
+# With mid-2024 Parsl, N>140 causes Parsl to hang
+N = 100
+
+# MAX_STACK is the maximum Python stack depth allowed for either
+# task submission to an executor or execution of a task.
+# With mid-2024 Parsl, 2-3 stack entries will be used per
+# recursively launched parsl task. So this should be smaller than
+# 2*N, but big enough to allow regular pytest+parsl stuff to
+# happen.
+MAX_STACK = 50
+
+
+def local_config():
+    return parsl.Config(executors=[ImmediateExecutor()])
+
+
+class ImmediateExecutor(ParslExecutor):
+    def start(self):
+        pass
+
+    def shutdown(self):
+        pass
+
+    def submit(self, func: Callable, resource_specification: Dict[str, Any], *args: Any, **kwargs: Any) -> Future:
+        stack_depth = len(inspect.stack())
+        assert stack_depth < MAX_STACK, "tasks should not be launched deep in the Python stack"
+        fut: Future[None] = Future()
+        res = func(*args, **kwargs)
+        fut.set_result(res)
+        return fut
+
+
+@parsl.python_app
+def chain(upstream):
+    stack_depth = len(inspect.stack())
+    assert stack_depth < MAX_STACK, "chained dependencies should not be launched deep in the Python stack"
+
+
+@pytest.mark.local
+def test_deep_dependency_stack_depth():
+
+    fut = Future()
+    here = fut
+
+    for _ in range(N):
+        here = chain(here)
+
+    fut.set_result(None)
+    here.result()

parsl/tests/test_python_apps/test_futures.py

@@ -11,9 +11,10 @@ Same applies to datafutures, and we need to know the behavior wrt.
 2. done() called on 1, vs 2
 
 """
-import pytest
 from os.path import basename
 
+import pytest
+
 from parsl.app.app import python_app
 from parsl.data_provider.files import File
 

parsl/tests/test_python_apps/test_join.py

@@ -3,7 +3,6 @@ import pytest
 from parsl import join_app, python_app
 from parsl.dataflow.errors import JoinError
 
-
 RESULT_CONSTANT = 3
 
 

parsl/tests/test_python_apps/test_lifted.py

@@ -1,10 +1,10 @@
+from concurrent.futures import Future
+from typing import TypeVar
+
 import pytest
 
-from concurrent.futures import Future
 from parsl import python_app
 
-from typing import TypeVar
-
 T = TypeVar('T')
 
 

parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py

@@ -1,4 +1,5 @@
 import pytest
+
 from parsl import python_app
 from parsl.dataflow.memoization import id_for_memo
 

parsl/tests/test_python_apps/test_pluggable_future_resolution.py

@@ -7,8 +7,8 @@ import pytest
 
 import parsl
 from parsl.config import Config
-from parsl.dataflow.errors import DependencyError
 from parsl.dataflow.dependency_resolvers import DEEP_DEPENDENCY_RESOLVER
+from parsl.dataflow.errors import DependencyError
 
 
 def local_config():

parsl/tests/test_radical/test_mpi_funcs.py

@@ -1,6 +1,6 @@
-import parsl
 import pytest
 
+import parsl
 from parsl.tests.configs.local_radical_mpi import fresh_config as local_config
 
 
@@ -16,7 +16,6 @@ def some_mpi_func(msg, sleep, comm=None, parsl_resource_specification={}):
 apps = []
 
 
-@pytest.mark.skip("hangs in CI - waiting for resolution of issue #3029")
 @pytest.mark.local
 @pytest.mark.radical
 def test_radical_mpi(n=7):

parsl/tests/test_regression/test_1480.py

@@ -1,5 +1,6 @@
-from parsl import python_app
 import pytest
+
+from parsl import python_app
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_regression/test_1653.py

@@ -1,5 +1,6 @@
-from parsl import python_app
 import pytest
+
+from parsl import python_app
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_regression/test_2652.py

@@ -1,4 +1,5 @@
 import pytest
+
 from parsl.jobs.states import JobState
 
 

parsl/tests/test_regression/test_69a.py

@@ -6,7 +6,6 @@ import pytest
 from parsl.app.app import bash_app, python_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/test_regression/test_854.py

@@ -1,8 +1,10 @@
-import time
 import multiprocessing
+import random
+import time
+
 import pytest
+
 from parsl.multiprocessing import MacSafeQueue
-import random
 
 
 def consumer(in_q, out_q, delay=0):

parsl/tests/test_regression/test_97_parallelism_0.py

@@ -1,11 +1,10 @@
 import pytest
 
 import parsl
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-from parsl.providers import LocalProvider
 from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 
 def local_config() -> Config:

parsl/tests/test_regression/test_98.py

@@ -6,7 +6,6 @@ import argparse
 import pytest
 
 import parsl
-
 from parsl.dataflow.dflow import DataFlowKernel
 from parsl.tests.configs.local_threads import config
 

parsl/tests/test_scaling/test_block_error_handler.py

@@ -1,11 +1,16 @@
+from functools import partial
+from unittest.mock import Mock
+
 import pytest
 
 from parsl.executors import HighThroughputExecutor
+from parsl.jobs.error_handlers import (
+    noop_error_handler,
+    simple_error_handler,
+    windowed_error_handler,
+)
+from parsl.jobs.states import JobState, JobStatus
 from parsl.providers import LocalProvider
-from unittest.mock import Mock
-from parsl.jobs.states import JobStatus, JobState
-from parsl.jobs.error_handlers import simple_error_handler, windowed_error_handler, noop_error_handler
-from functools import partial
 
 
 @pytest.mark.local

parsl/tests/test_scaling/test_scale_down.py

@@ -4,13 +4,12 @@ import time
 import pytest
 
 import parsl
-
 from parsl import File, python_app
-from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
 
 logger = logging.getLogger(__name__)
 

parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py

@@ -1,15 +1,14 @@
+from threading import Event
+
 import pytest
 
 import parsl
-
 from parsl import File, python_app
-from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-
-from threading import Event
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
 
 _max_blocks = 5
 _min_blocks = 0

parsl/tests/test_scaling/test_scale_down_htex_unregistered.py

@@ -4,14 +4,13 @@ import time
 import pytest
 
 import parsl
-
 from parsl import File, python_app
-from parsl.jobs.states import JobState, TERMINAL_STATES
-from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.jobs.states import TERMINAL_STATES, JobState
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
 
 logger = logging.getLogger(__name__)
 

parsl/tests/test_scaling/test_shutdown_scalein.py

@@ -1,3 +1,4 @@
+import random
 import threading
 
 import pytest
@@ -9,8 +10,6 @@ from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
 from parsl.providers import LocalProvider
 
-import random
-
 # we need some blocks, but it doesn't matter too much how many, as long
 # as they can all start up and get registered within the try_assert
 # timeout later on.

parsl/tests/test_serialization/test_2555_caching_deserializer.py

@@ -1,6 +1,6 @@
-import parsl
 import pytest
 
+import parsl
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_serialization/test_basic.py

@@ -1,5 +1,6 @@
 import pytest
-from parsl.serialize import serialize, deserialize
+
+from parsl.serialize import deserialize, serialize
 from parsl.serialize.concretes import DillSerializer, PickleSerializer
 
 

parsl/tests/test_serialization/test_htex_code_cache.py

@@ -1,10 +1,9 @@
-import parsl
-import pytest
-
 from typing import Any
 
-from parsl.serialize.facade import methods_for_code
+import pytest
 
+import parsl
+from parsl.serialize.facade import methods_for_code
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_serialization/test_pack_resource_spec.py

@@ -1,5 +1,6 @@
 import pytest
-from parsl.serialize import unpack_res_spec_apply_message, pack_res_spec_apply_message
+
+from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message
 
 
 def double(x: int, y: int = 2) -> int:

parsl/tests/test_serialization/test_proxystore_configured.py

@@ -1,19 +1,24 @@
 import logging
-import pytest
 import uuid
 
+import pytest
+
 import parsl
-from parsl.serialize.facade import additional_methods_for_deserialization, methods_for_data, register_method_for_data
+from parsl.serialize.facade import (
+    additional_methods_for_deserialization,
+    methods_for_data,
+    register_method_for_data,
+)
 from parsl.tests.configs.htex_local import fresh_config
 
-
 logger = logging.getLogger(__name__)
 
 
 def local_setup():
-    from parsl.serialize.proxystore import ProxyStoreSerializer
-    from proxystore.store import Store, register_store
     from proxystore.connectors.file import FileConnector
+    from proxystore.store import Store, register_store
+
+    from parsl.serialize.proxystore import ProxyStoreSerializer
 
     parsl.load(fresh_config())
 

parsl/tests/test_serialization/test_proxystore_impl.py

@@ -1,6 +1,7 @@
-import pytest
 import uuid
 
+import pytest
+
 
 def policy_example(o):
     """Example policy will proxy only lists."""
@@ -13,10 +14,11 @@ def test_proxystore_nonglobal():
     """
     # import in function, because proxystore is not importable in base parsl
     # installation.
-    from parsl.serialize.proxystore import ProxyStoreSerializer
+    from proxystore.connectors.file import FileConnector
     from proxystore.proxy import Proxy
     from proxystore.store import Store, register_store
-    from proxystore.connectors.file import FileConnector
+
+    from parsl.serialize.proxystore import ProxyStoreSerializer
 
     store = Store(name='parsl_store_' + str(uuid.uuid4()), connector=FileConnector(store_dir="/tmp"))
     register_store(store)

parsl/tests/test_shutdown/test_kill_monitoring.py

@@ -1,9 +1,10 @@
 import os
-import parsl
-import pytest
 import signal
 import time
 
+import pytest
+
+import parsl
 from parsl.tests.configs.htex_local_alternate import fresh_config
 
 # This is a very generous upper bound on expected shutdown time of target

parsl/tests/test_staging/staging_provider.py

@@ -62,8 +62,8 @@ def make_stage_out_app(executor, dfk):
 
 
 def stage_out_noop(app_fu, inputs=[], _parsl_staging_inhibit=True):
-    import time
     import logging
+    import time
     logger = logging.getLogger(__name__)
     logger.info("stage_out_noop")
     time.sleep(1)
@@ -75,8 +75,8 @@ def make_stage_in_app(executor, dfk):
 
 
 def stage_in_noop(parent_fut=None, outputs=[], _parsl_staging_inhibit=True):
-    import time
     import logging
+    import time
     logger = logging.getLogger(__name__)
     logger.info("stage_in_noop")
     time.sleep(1)

parsl/tests/test_staging/test_1316.py

@@ -1,8 +1,9 @@
+import time
+
 import pytest
 
 import parsl
-import time
-from parsl import python_app, ThreadPoolExecutor
+from parsl import ThreadPoolExecutor, python_app
 from parsl.config import Config
 from parsl.data_provider.files import File
 from parsl.data_provider.staging import Staging

parsl/tests/test_staging/test_docs_1.py

@@ -1,6 +1,6 @@
 import pytest
 
-from parsl import python_app, File
+from parsl import File, python_app
 
 
 @python_app

parsl/tests/test_staging/test_docs_2.py

@@ -1,5 +1,6 @@
 import pytest
-from parsl import bash_app, File
+
+from parsl import File, bash_app
 from parsl.tests.configs.local_threads import fresh_config as local_config