parsl 2024.5.13__py3-none-any.whl → 2024.5.27__py3-none-any.whl

parsl/tests/test_monitoring/test_memoization_representation.py

@@ -47,7 +47,6 @@ def test_hashsum():
 
     logger.info("cleaning up parsl")
     parsl.dfk().cleanup()
-    parsl.clear()
 
     # at this point, we should find one row in the monitoring database.
 

parsl/tests/test_monitoring/test_stdouterr.py

@@ -103,8 +103,6 @@ def test_stdstream_to_monitoring(stdx, expected_stdx, stream, tmpd_cwd, caplog):
         kwargs = {stream: stdx}
         stdapp(**kwargs).result()
 
-    parsl.clear()
-
     engine = sqlalchemy.create_engine(c.monitoring.logging_endpoint)
     with engine.begin() as connection:
 

parsl/tests/test_mpi_apps/test_bad_mpi_config.py

@@ -2,15 +2,15 @@ import pytest
 
 from parsl import Config
 from parsl.executors import HighThroughputExecutor
-from parsl.launchers import SrunLauncher, SingleNodeLauncher, SimpleLauncher, AprunLauncher
+from parsl.launchers import SrunLauncher, AprunLauncher, SimpleLauncher
 from parsl.providers import SlurmProvider
 
 
 @pytest.mark.local
 def test_bad_launcher_with_mpi_mode():
-    """AssertionError if a launcher other than SingleNodeLauncher is supplied"""
+    """AssertionError if a launcher other than SimpleLauncher is supplied"""
 
-    for launcher in [SrunLauncher(), SimpleLauncher(), AprunLauncher()]:
+    for launcher in [SrunLauncher(), AprunLauncher()]:
         with pytest.raises(AssertionError):
             Config(executors=[
                 HighThroughputExecutor(
@@ -22,20 +22,12 @@ def test_bad_launcher_with_mpi_mode():
 
 @pytest.mark.local
 def test_correct_launcher_with_mpi_mode():
-    """Confirm that SingleNodeLauncer works with mpi_mode"""
+    """Confirm that SimpleLauncher works with mpi_mode"""
 
     config = Config(executors=[
         HighThroughputExecutor(
             enable_mpi_mode=True,
-            provider=SlurmProvider(launcher=SingleNodeLauncher()),
+            provider=SlurmProvider(launcher=SimpleLauncher()),
         )
     ])
-    assert isinstance(config.executors[0].provider.launcher, SingleNodeLauncher)
-
-    config = Config(executors=[
-        HighThroughputExecutor(
-            enable_mpi_mode=True,
-            provider=SlurmProvider(),
-        )
-    ])
-    assert isinstance(config.executors[0].provider.launcher, SingleNodeLauncher)
+    assert isinstance(config.executors[0].provider.launcher, SimpleLauncher)

parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py

@@ -1,4 +1,3 @@
-import logging
 from typing import Dict
 import pytest
 import parsl
@@ -8,17 +7,12 @@ from parsl.tests.configs.htex_local import fresh_config
 EXECUTOR_LABEL = "MPI_TEST"
 
 
-def local_setup():
+def local_config():
     config = fresh_config()
     config.executors[0].label = EXECUTOR_LABEL
     config.executors[0].max_workers_per_node = 1
     config.executors[0].enable_mpi_mode = False
-    parsl.load(config)
-
-
-def local_teardown():
-    parsl.dfk().cleanup()
-    parsl.clear()
+    return config
 
 
 @python_app

parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py

@@ -6,6 +6,8 @@ import parsl
 from parsl import python_app, bash_app
 from parsl.tests.configs.htex_local import fresh_config
 
+from parsl.executors.high_throughput.mpi_prefix_composer import MissingResourceSpecification
+
 import os
 
 EXECUTOR_LABEL = "MPI_TEST"
@@ -28,7 +30,6 @@ def local_setup():
 
 def local_teardown():
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app
@@ -169,3 +170,11 @@ def test_simulated_load(rounds: int = 100):
         total_ranks, nodes = future.result(timeout=10)
         assert len(nodes) == futures[future]["num_nodes"]
         assert total_ranks == futures[future]["num_nodes"] * futures[future]["ranks_per_node"]
+
+
+@pytest.mark.local
+def test_missing_resource_spec():
+
+    with pytest.raises(MissingResourceSpecification):
+        future = mock_app(sleep_dur=0.4)
+        future.result(timeout=10)

parsl/tests/test_mpi_apps/test_mpiex.py

@@ -0,0 +1,64 @@
+"""Tests for the wrapper class"""
+from inspect import signature
+from pathlib import Path
+
+import pytest
+
+import parsl
+from .test_mpi_mode_enabled import get_env_vars
+from parsl import HighThroughputExecutor, Config
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+from parsl.executors.high_throughput.mpi_executor import MPIExecutor
+
+cwd = Path(__file__).parent.absolute()
+pbs_nodefile = cwd.joinpath("mocks", "pbs_nodefile")
+
+
+def local_config():
+    return Config(
+        executors=[
+            MPIExecutor(
+                max_workers_per_block=1,
+                provider=LocalProvider(
+                    worker_init=f"export PBS_NODEFILE={pbs_nodefile}",
+                    launcher=SimpleLauncher()
+                )
+            )
+        ]
+    )
+
+
+@pytest.mark.local
+def test_docstring():
+    """Ensure the old kwargs are copied over into the new class"""
+    assert 'label' in MPIExecutor.__doc__
+    assert 'max_workers_per_block' in MPIExecutor.__doc__
+    assert 'available_accelerators' not in MPIExecutor.__doc__
+
+
+@pytest.mark.local
+def test_init():
+    """Ensure all relevant kwargs are copied over from HTEx"""
+
+    new_kwargs = {'max_workers_per_block'}
+    excluded_kwargs = {'available_accelerators', 'enable_mpi_mode', 'cores_per_worker', 'max_workers_per_node',
+                       'mem_per_worker', 'cpu_affinity', 'max_workers'}
+
+    # Get the kwargs from both HTEx and MPIEx
+    htex_kwargs = set(signature(HighThroughputExecutor.__init__).parameters)
+    mpix_kwargs = set(signature(MPIExecutor.__init__).parameters)
+
+    assert mpix_kwargs.difference(htex_kwargs) == new_kwargs
+    assert len(mpix_kwargs.intersection(excluded_kwargs)) == 0
+    assert mpix_kwargs.union(excluded_kwargs).difference(new_kwargs) == htex_kwargs
+
+
+@pytest.mark.local
+def test_get_env():
+    future = get_env_vars(parsl_resource_specification={
+        "num_nodes": 3,
+        "ranks_per_node": 5,
+    })
+    env_vars = future.result()
+    assert env_vars['PARSL_NUM_RANKS'] == '15'

parsl/tests/test_mpi_apps/test_resource_spec.py

@@ -19,7 +19,8 @@ from parsl.executors.high_throughput.mpi_resource_management import (
 )
 from parsl.executors.high_throughput.mpi_prefix_composer import (
     validate_resource_spec,
-    InvalidResourceSpecification
+    InvalidResourceSpecification,
+    MissingResourceSpecification
 )
 
 EXECUTOR_LABEL = "MPI_TEST"
@@ -122,18 +123,22 @@ def test_top_level():
 
 @pytest.mark.local
 @pytest.mark.parametrize(
-    "resource_spec, exception",
+    "resource_spec, is_mpi_enabled, exception",
     (
-        ({"num_nodes": 2, "ranks_per_node": 1}, None),
-        ({"launcher_options": "--debug_foo"}, None),
-        ({"num_nodes": 2, "BAD_OPT": 1}, InvalidResourceSpecification),
-        ({}, None),
+        ({"num_nodes": 2, "ranks_per_node": 1}, False, None),
+        ({"launcher_options": "--debug_foo"}, False, None),
+        ({"num_nodes": 2, "BAD_OPT": 1}, False, InvalidResourceSpecification),
+        ({}, False, None),
+        ({"num_nodes": 2, "ranks_per_node": 1}, True, None),
+        ({"launcher_options": "--debug_foo"}, True, None),
+        ({"num_nodes": 2, "BAD_OPT": 1}, True, InvalidResourceSpecification),
+        ({}, True, MissingResourceSpecification),
     )
 )
-def test_resource_spec(resource_spec: Dict, exception):
+def test_resource_spec(resource_spec: Dict, is_mpi_enabled: bool, exception):
     if exception:
         with pytest.raises(exception):
-            validate_resource_spec(resource_spec)
+            validate_resource_spec(resource_spec, is_mpi_enabled)
     else:
-        result = validate_resource_spec(resource_spec)
+        result = validate_resource_spec(resource_spec, is_mpi_enabled)
         assert result is None

parsl/tests/test_python_apps/test_context_manager.py

@@ -15,14 +15,6 @@ def foo(x, stdout='foo.stdout'):
     return f"echo {x + 1}"
 
 
-def local_setup():
-    pass
-
-
-def local_teardown():
-    parsl.clear()
-
-
 @pytest.mark.local
 def test_within_context_manger(tmpd_cwd):
     config = fresh_config()
@@ -37,4 +29,4 @@ def test_within_context_manger(tmpd_cwd):
 
     with pytest.raises(NoDataFlowKernelError) as excinfo:
         square(2).result()
-    assert str(excinfo.value) == "Cannot submit to a DFK that has been cleaned up"
+    assert str(excinfo.value) == "Must first load config"

parsl/tests/test_python_apps/test_lifted.py

@@ -3,24 +3,28 @@ import pytest
 from concurrent.futures import Future
 from parsl import python_app
 
+from typing import TypeVar
+
+T = TypeVar('T')
+
 
 @python_app
-def returns_a_dict():
+def returns_a_dict() -> dict:
     return {"a": "X", "b": "Y"}
 
 
 @python_app
-def returns_a_list():
+def returns_a_list() -> list:
     return ["X", "Y"]
 
 
 @python_app
-def returns_a_tuple():
+def returns_a_tuple() -> tuple:
     return ("X", "Y")
 
 
 @python_app
-def returns_a_class():
+def returns_a_class() -> type:
     from dataclasses import dataclass
 
     @dataclass
@@ -38,7 +42,7 @@ class MyOuterClass():
 
 
 @python_app
-def returns_a_class_instance():
+def returns_a_class_instance() -> object:
     return MyOuterClass()
 
 
@@ -110,7 +114,7 @@ def test_returns_a_class():
 
 
 @python_app
-def passthrough(v):
+def passthrough(v: T) -> T:
     return v
 
 

parsl/tests/test_python_apps/test_pluggable_future_resolution.py

@@ -0,0 +1,161 @@
+from concurrent.futures import Future
+from pathlib import Path
+from threading import Event
+from typing import Sequence
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.dataflow.errors import DependencyError
+from parsl.dataflow.dependency_resolvers import DEEP_DEPENDENCY_RESOLVER
+
+
+def local_config():
+    return Config(dependency_resolver=DEEP_DEPENDENCY_RESOLVER)
+
+
+@parsl.python_app
+def a(event):
+    event.wait()
+    return 7
+
+
+@parsl.python_app
+def b(x: int):
+    return x + 1
+
+
+@pytest.mark.local
+def test_simple_pos_arg():
+    e = Event()
+    s = a(e)
+    f_b = b(s)
+    e.set()
+
+    assert f_b.result() == 8
+
+
+@parsl.python_app
+def b_first(x: Sequence[int]):
+    return x[0] + 1
+
+
+@pytest.mark.local
+def test_tuple_pos_arg():
+    e = Event()
+    s = (a(e),)
+    f_b = b_first(s)
+    e.set()
+    assert f_b.result() == 8
+
+
+@pytest.mark.local
+def test_list_exception():
+    a = Future()
+    a.set_exception(RuntimeError("artificial error"))
+    f_b = b([a])
+    assert isinstance(f_b.exception(), DependencyError)
+
+
+@parsl.python_app
+def make_path(s: str):
+    return Path(s)
+
+
+@parsl.python_app
+def append_paths(iterable, end_str: str = "end"):
+    type_ = type(iterable)
+    return type_([Path(s, end_str) for s in iterable])
+
+
+@pytest.mark.local
+@pytest.mark.parametrize(
+    "type_",
+    [
+        tuple,
+        list,
+        set,
+    ],
+)
+def test_resolving_iterables(type_):
+    output1 = make_path("test1")
+    output2 = make_path("test2")
+    output3 = append_paths(type_([output1, output2]), end_str="end")
+    assert output3.result() == type_([Path("test1", "end"), Path("test2", "end")])
+
+
+@parsl.python_app
+def append_paths_dict(iterable: dict, end_str: str = "end"):
+    return {Path(k, end_str): Path(v, end_str) for k, v in iterable.items()}
+
+
+@pytest.mark.local
+def test_resolving_dict():
+    output1 = make_path("test1")
+    output2 = make_path("test2")
+    output3 = append_paths_dict({output1: output2}, end_str="end")
+    assert output3.result() == {Path("test1", "end"): Path("test2", "end")}
+
+
+@parsl.python_app
+def extract_deep(struct: list):
+    return struct[0][0][0][0][0]
+
+
+@pytest.mark.local
+def test_deeper_list():
+    f = Future()
+    f.set_result(7)
+    f_b = extract_deep([[[[[f]]]]])
+
+    assert f_b.result() == 7
+
+
+@pytest.mark.local
+def test_deeper_list_and_tuple():
+    f = Future()
+    f.set_result(7)
+    f_b = extract_deep([([([f],)],)])
+
+    assert f_b.result() == 7
+
+
+@parsl.python_app
+def dictionary_checker(d):
+    assert d["a"] == [30, 10]
+    assert d["b"] == 20
+
+
+@pytest.mark.local
+def test_dictionary():
+    k1 = Future()
+    k1.set_result("a")
+    k2 = Future()
+    k2.set_result("b")
+    v1 = Future()
+    v1.set_result(10)
+
+    # this .result() will fail if the asserts fail
+    dictionary_checker({k1: [30, v1], k2: 20}).result()
+
+
+@pytest.mark.local
+def test_dictionary_later():
+    k1 = Future()
+    k2 = Future()
+    v1 = Future()
+
+    f1 = dictionary_checker({k1: [30, v1], k2: 20})
+
+    assert not f1.done()
+    k1.set_result("a")
+    k2.set_result("b")
+    v1.set_result(10)
+
+    # having set the results, f1 should fairly rapidly complete (but not
+    # instantly) so we can't assert on done() here... but we can
+    # check that f1 does "eventually" complete... meaning that
+    # none of the assertions inside test_dictionary have failed
+
+    assert f1.result() is None

parsl/tests/test_scaling/test_regression_1621.py

@@ -69,6 +69,4 @@ def test_one_block(tmpd_cwd):
     with parsl.load(config):
         app().result()
 
-    parsl.clear()
-
     assert oneshot_provider.recorded_submits == 1

parsl/tests/test_scaling/test_shutdown_scalein.py

@@ -72,7 +72,5 @@ def test_shutdown_scalein_blocks(tmpd_cwd, try_assert):
         # this will wait for everything to be scaled out fully
         try_assert(lambda: len(htex.connected_managers()) == BLOCK_COUNT)
 
-    parsl.clear()
-
     assert len(accumulating_provider.submit_job_ids) == BLOCK_COUNT, f"Exactly {BLOCK_COUNT} blocks should have been launched"
     assert len(accumulating_provider.cancel_job_ids) == BLOCK_COUNT, f"Exactly {BLOCK_COUNT} blocks should have been scaled in"

parsl/tests/test_serialization/test_proxystore_configured.py

@@ -38,7 +38,6 @@ def local_setup():
 
 def local_teardown():
     parsl.dfk().cleanup()
-    parsl.clear()
 
     methods_for_data.clear()
     methods_for_data.update(previous_methods)

parsl/tests/test_shutdown/test_kill_monitoring.py

@@ -25,7 +25,6 @@ def test_no_kills():
     assert parsl.dfk().monitoring is not None, "This test requires monitoring"
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @pytest.mark.local
@@ -62,4 +61,3 @@ def test_kill_monitoring_helper_process(sig, process_attr, try_assert):
     simple_app().result()
 
     parsl.dfk().cleanup()
-    parsl.clear()

parsl/tests/test_staging/test_1316.py

@@ -60,7 +60,6 @@ def test_1316_local_path_on_execution_side_sp2():
     assert not file.local_path, "The local_path on the submit side should not be set"
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @pytest.mark.local
@@ -83,4 +82,3 @@ def test_1316_local_path_setting_preserves_dependency_sp2():
     assert not file.local_path, "The local_path on the submit side should not be set"
 
     parsl.dfk().cleanup()
-    parsl.clear()

parsl/tests/test_staging/test_elaborate_noop_file.py

@@ -44,7 +44,6 @@ def storage_access_parsl():
     yield _setup_config
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @pytest.mark.local

parsl/tests/test_summary.py

@@ -22,7 +22,6 @@ def test_summary(caplog):
     fail().exception()
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
     assert "Summary of tasks in DFK:" in caplog.text
     assert "Tasks in state States.exec_done: 1" in caplog.text

parsl/tests/test_threads/test_configs.py

@@ -30,5 +30,4 @@ def test_parallel_for():
     assert thread_count <= config.executors[0].max_threads, "More threads than allowed"
     assert process_count == 1, "More processes than allowed"
     dfk.cleanup()
-    parsl.clear()
     return d

parsl/tests/test_threads/test_lazy_errors.py

@@ -24,5 +24,4 @@ def test_lazy_behavior():
         assert f.done()
 
     parsl.dfk().cleanup()
-    parsl.clear()
     return

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.05.13'
+VERSION = '2024.05.27'

{parsl-2024.5.13.dist-info → parsl-2024.5.27.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.5.13
+Version: 2024.5.27
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.05.13.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.05.27.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -55,7 +55,8 @@ Requires-Dist: pyyaml ; extra == 'all'
 Requires-Dist: cffi ; extra == 'all'
 Requires-Dist: jsonschema ; extra == 'all'
 Requires-Dist: proxystore ; extra == 'all'
-Requires-Dist: radical.pilot ==1.52.1 ; extra == 'all'
+Requires-Dist: radical.pilot ==1.60 ; extra == 'all'
+Requires-Dist: radical.utils ==1.60 ; extra == 'all'
 Provides-Extra: aws
 Requires-Dist: boto3 ; extra == 'aws'
 Provides-Extra: azure
@@ -84,7 +85,8 @@ Requires-Dist: oauth-ssh >=0.9 ; extra == 'oauth_ssh'
 Provides-Extra: proxystore
 Requires-Dist: proxystore ; extra == 'proxystore'
 Provides-Extra: radical-pilot
-Requires-Dist: radical.pilot ==1.52.1 ; extra == 'radical-pilot'
+Requires-Dist: radical.pilot ==1.60 ; extra == 'radical-pilot'
+Requires-Dist: radical.utils ==1.60 ; extra == 'radical-pilot'
 Provides-Extra: visualization
 Requires-Dist: pydot ; extra == 'visualization'
 Requires-Dist: networkx <2.6,>=2.5 ; extra == 'visualization'