parsl 2024.5.20__py3-none-any.whl → 2024.6.3__py3-none-any.whl

parsl/tests/test_mpi_apps/test_bad_mpi_config.py

@@ -2,7 +2,7 @@ import pytest
 
 from parsl import Config
 from parsl.executors import HighThroughputExecutor
-from parsl.launchers import SrunLauncher, AprunLauncher, SimpleLauncher
+from parsl.launchers import AprunLauncher, SimpleLauncher, SrunLauncher
 from parsl.providers import SlurmProvider
 
 

parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py

@@ -1,5 +1,7 @@
 from typing import Dict
+
 import pytest
+
 import parsl
 from parsl import python_app
 from parsl.tests.configs.htex_local import fresh_config
@@ -7,17 +9,12 @@ from parsl.tests.configs.htex_local import fresh_config
 EXECUTOR_LABEL = "MPI_TEST"
 
 
-def local_setup():
+def local_config():
     config = fresh_config()
     config.executors[0].label = EXECUTOR_LABEL
     config.executors[0].max_workers_per_node = 1
     config.executors[0].enable_mpi_mode = False
-    parsl.load(config)
-
-
-def local_teardown():
-    parsl.dfk().cleanup()
-    parsl.clear()
+    return config
 
 
 @python_app

parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py

@@ -1,13 +1,17 @@
 import logging
+import os
 import random
 from typing import Dict
+
 import pytest
+
 import parsl
-from parsl import python_app, bash_app
+from parsl import bash_app, python_app
+from parsl.executors.high_throughput.mpi_prefix_composer import (
+    MissingResourceSpecification,
+)
 from parsl.tests.configs.htex_local import fresh_config
 
-import os
-
 EXECUTOR_LABEL = "MPI_TEST"
 
 
@@ -28,7 +32,6 @@ def local_setup():
 
 def local_teardown():
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @python_app
@@ -169,3 +172,11 @@ def test_simulated_load(rounds: int = 100):
         total_ranks, nodes = future.result(timeout=10)
         assert len(nodes) == futures[future]["num_nodes"]
         assert total_ranks == futures[future]["num_nodes"] * futures[future]["ranks_per_node"]
+
+
+@pytest.mark.local
+def test_missing_resource_spec():
+
+    with pytest.raises(MissingResourceSpecification):
+        future = mock_app(sleep_dur=0.4)
+        future.result(timeout=10)

parsl/tests/test_mpi_apps/test_mpi_prefix.py

@@ -1,14 +1,14 @@
 import logging
+
 import pytest
 
-from parsl.executors.high_throughput.mpi_resource_management import Scheduler
 from parsl.executors.high_throughput.mpi_prefix_composer import (
-    compose_srun_launch_cmd,
+    compose_all,
     compose_aprun_launch_cmd,
     compose_mpiexec_launch_cmd,
-    compose_all,
+    compose_srun_launch_cmd,
 )
-
+from parsl.executors.high_throughput.mpi_resource_management import Scheduler
 
 resource_spec = {"num_nodes": 2,
                  "num_ranks": 8,

parsl/tests/test_mpi_apps/test_mpi_scheduler.py

@@ -1,9 +1,14 @@
 import logging
 import os
+import pickle
 from unittest import mock
+
 import pytest
-import pickle
-from parsl.executors.high_throughput.mpi_resource_management import TaskScheduler, MPITaskScheduler
+
+from parsl.executors.high_throughput.mpi_resource_management import (
+    MPITaskScheduler,
+    TaskScheduler,
+)
 from parsl.multiprocessing import SpawnContext
 from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message
 

parsl/tests/test_mpi_apps/test_mpiex.py

@@ -5,11 +5,12 @@ from pathlib import Path
 import pytest
 
 import parsl
-from .test_mpi_mode_enabled import get_env_vars
-from parsl import HighThroughputExecutor, Config
+from parsl import Config, HighThroughputExecutor
+from parsl.executors.high_throughput.mpi_executor import MPIExecutor
 from parsl.launchers import SimpleLauncher
 from parsl.providers import LocalProvider
-from parsl.executors.high_throughput.mpi_executor import MPIExecutor
+
+from .test_mpi_mode_enabled import get_env_vars
 
 cwd = Path(__file__).parent.absolute()
 pbs_nodefile = cwd.joinpath("mocks", "pbs_nodefile")

parsl/tests/test_mpi_apps/test_resource_spec.py

@@ -2,25 +2,25 @@ import contextlib
 import logging
 import os
 import typing
-
+import unittest
+from typing import Dict
 
 import pytest
-import unittest
 
 import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.htex_local import fresh_config
-from typing import Dict
+from parsl.executors.high_throughput.mpi_prefix_composer import (
+    InvalidResourceSpecification,
+    MissingResourceSpecification,
+    validate_resource_spec,
+)
 from parsl.executors.high_throughput.mpi_resource_management import (
+    get_nodes_in_batchjob,
     get_pbs_hosts_list,
     get_slurm_hosts_list,
-    get_nodes_in_batchjob,
     identify_scheduler,
 )
-from parsl.executors.high_throughput.mpi_prefix_composer import (
-    validate_resource_spec,
-    InvalidResourceSpecification
-)
+from parsl.tests.configs.htex_local import fresh_config
 
 EXECUTOR_LABEL = "MPI_TEST"
 
@@ -122,18 +122,22 @@ def test_top_level():
 
 @pytest.mark.local
 @pytest.mark.parametrize(
-    "resource_spec, exception",
+    "resource_spec, is_mpi_enabled, exception",
     (
-        ({"num_nodes": 2, "ranks_per_node": 1}, None),
-        ({"launcher_options": "--debug_foo"}, None),
-        ({"num_nodes": 2, "BAD_OPT": 1}, InvalidResourceSpecification),
-        ({}, None),
+        ({"num_nodes": 2, "ranks_per_node": 1}, False, None),
+        ({"launcher_options": "--debug_foo"}, False, None),
+        ({"num_nodes": 2, "BAD_OPT": 1}, False, InvalidResourceSpecification),
+        ({}, False, None),
+        ({"num_nodes": 2, "ranks_per_node": 1}, True, None),
+        ({"launcher_options": "--debug_foo"}, True, None),
+        ({"num_nodes": 2, "BAD_OPT": 1}, True, InvalidResourceSpecification),
+        ({}, True, MissingResourceSpecification),
     )
 )
-def test_resource_spec(resource_spec: Dict, exception):
+def test_resource_spec(resource_spec: Dict, is_mpi_enabled: bool, exception):
     if exception:
         with pytest.raises(exception):
-            validate_resource_spec(resource_spec)
+            validate_resource_spec(resource_spec, is_mpi_enabled)
     else:
-        result = validate_resource_spec(resource_spec)
+        result = validate_resource_spec(resource_spec, is_mpi_enabled)
         assert result is None

parsl/tests/test_providers/test_cobalt_deprecation_warning.py

@@ -1,5 +1,7 @@
 import warnings
+
 import pytest
+
 from parsl.providers import CobaltProvider
 
 

parsl/tests/test_providers/test_local_provider.py

@@ -1,7 +1,6 @@
 import logging
 import os
 import pathlib
-import pytest
 import random
 import shutil
 import socket
@@ -10,6 +9,8 @@ import tempfile
 import threading
 import time
 
+import pytest
+
 from parsl.channels import LocalChannel, SSHChannel
 from parsl.jobs.states import JobState
 from parsl.launchers import SingleNodeLauncher

parsl/tests/test_providers/test_pbspro_template.py

@@ -1,6 +1,6 @@
 import random
-
 from unittest import mock
+
 import pytest
 
 from parsl.channels import LocalChannel

parsl/tests/test_providers/test_slurm_template.py

@@ -1,7 +1,7 @@
 import logging
 import random
-
 from unittest import mock
+
 import pytest
 
 from parsl.channels import LocalChannel

parsl/tests/test_providers/test_submiterror_deprecation.py

@@ -1,7 +1,8 @@
-import pytest
 import random
 import string
 
+import pytest
+
 from parsl.providers.errors import SubmitException
 
 

parsl/tests/test_python_apps/test_context_manager.py

@@ -1,8 +1,9 @@
-import parsl
-from parsl.tests.configs.local_threads import fresh_config
 import pytest
-from parsl.errors import NoDataFlowKernelError
+
+import parsl
 from parsl.dataflow.dflow import DataFlowKernel
+from parsl.errors import NoDataFlowKernelError
+from parsl.tests.configs.local_threads import fresh_config
 
 
 @parsl.python_app
@@ -15,14 +16,6 @@ def foo(x, stdout='foo.stdout'):
     return f"echo {x + 1}"
 
 
-def local_setup():
-    pass
-
-
-def local_teardown():
-    parsl.clear()
-
-
 @pytest.mark.local
 def test_within_context_manger(tmpd_cwd):
     config = fresh_config()
@@ -37,4 +30,4 @@ def test_within_context_manger(tmpd_cwd):
 
     with pytest.raises(NoDataFlowKernelError) as excinfo:
         square(2).result()
-    assert str(excinfo.value) == "Cannot submit to a DFK that has been cleaned up"
+    assert str(excinfo.value) == "Must first load config"

parsl/tests/test_python_apps/test_dep_standard_futures.py

@@ -1,6 +1,7 @@
+from concurrent.futures import Future
+
 import parsl
 from parsl.dataflow.errors import DependencyError
-from concurrent.futures import Future
 
 
 @parsl.python_app

parsl/tests/test_python_apps/test_futures.py

@@ -11,9 +11,10 @@ Same applies to datafutures, and we need to know the behavior wrt.
 2. done() called on 1, vs 2
 
 """
-import pytest
 from os.path import basename
 
+import pytest
+
 from parsl.app.app import python_app
 from parsl.data_provider.files import File
 

parsl/tests/test_python_apps/test_join.py

@@ -3,7 +3,6 @@ import pytest
 from parsl import join_app, python_app
 from parsl.dataflow.errors import JoinError
 
-
 RESULT_CONSTANT = 3
 
 

parsl/tests/test_python_apps/test_lifted.py

@@ -1,26 +1,30 @@
+from concurrent.futures import Future
+from typing import TypeVar
+
 import pytest
 
-from concurrent.futures import Future
 from parsl import python_app
 
+T = TypeVar('T')
+
 
 @python_app
-def returns_a_dict():
+def returns_a_dict() -> dict:
     return {"a": "X", "b": "Y"}
 
 
 @python_app
-def returns_a_list():
+def returns_a_list() -> list:
     return ["X", "Y"]
 
 
 @python_app
-def returns_a_tuple():
+def returns_a_tuple() -> tuple:
     return ("X", "Y")
 
 
 @python_app
-def returns_a_class():
+def returns_a_class() -> type:
     from dataclasses import dataclass
 
     @dataclass
@@ -38,7 +42,7 @@ class MyOuterClass():
 
 
 @python_app
-def returns_a_class_instance():
+def returns_a_class_instance() -> object:
     return MyOuterClass()
 
 
@@ -110,7 +114,7 @@ def test_returns_a_class():
 
 
 @python_app
-def passthrough(v):
+def passthrough(v: T) -> T:
     return v
 
 

parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py

@@ -1,4 +1,5 @@
 import pytest
+
 from parsl import python_app
 from parsl.dataflow.memoization import id_for_memo
 

parsl/tests/test_python_apps/test_pluggable_future_resolution.py

@@ -0,0 +1,161 @@
+from concurrent.futures import Future
+from pathlib import Path
+from threading import Event
+from typing import Sequence
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.dataflow.dependency_resolvers import DEEP_DEPENDENCY_RESOLVER
+from parsl.dataflow.errors import DependencyError
+
+
+def local_config():
+    return Config(dependency_resolver=DEEP_DEPENDENCY_RESOLVER)
+
+
+@parsl.python_app
+def a(event):
+    event.wait()
+    return 7
+
+
+@parsl.python_app
+def b(x: int):
+    return x + 1
+
+
+@pytest.mark.local
+def test_simple_pos_arg():
+    e = Event()
+    s = a(e)
+    f_b = b(s)
+    e.set()
+
+    assert f_b.result() == 8
+
+
+@parsl.python_app
+def b_first(x: Sequence[int]):
+    return x[0] + 1
+
+
+@pytest.mark.local
+def test_tuple_pos_arg():
+    e = Event()
+    s = (a(e),)
+    f_b = b_first(s)
+    e.set()
+    assert f_b.result() == 8
+
+
+@pytest.mark.local
+def test_list_exception():
+    a = Future()
+    a.set_exception(RuntimeError("artificial error"))
+    f_b = b([a])
+    assert isinstance(f_b.exception(), DependencyError)
+
+
+@parsl.python_app
+def make_path(s: str):
+    return Path(s)
+
+
+@parsl.python_app
+def append_paths(iterable, end_str: str = "end"):
+    type_ = type(iterable)
+    return type_([Path(s, end_str) for s in iterable])
+
+
+@pytest.mark.local
+@pytest.mark.parametrize(
+    "type_",
+    [
+        tuple,
+        list,
+        set,
+    ],
+)
+def test_resolving_iterables(type_):
+    output1 = make_path("test1")
+    output2 = make_path("test2")
+    output3 = append_paths(type_([output1, output2]), end_str="end")
+    assert output3.result() == type_([Path("test1", "end"), Path("test2", "end")])
+
+
+@parsl.python_app
+def append_paths_dict(iterable: dict, end_str: str = "end"):
+    return {Path(k, end_str): Path(v, end_str) for k, v in iterable.items()}
+
+
+@pytest.mark.local
+def test_resolving_dict():
+    output1 = make_path("test1")
+    output2 = make_path("test2")
+    output3 = append_paths_dict({output1: output2}, end_str="end")
+    assert output3.result() == {Path("test1", "end"): Path("test2", "end")}
+
+
+@parsl.python_app
+def extract_deep(struct: list):
+    return struct[0][0][0][0][0]
+
+
+@pytest.mark.local
+def test_deeper_list():
+    f = Future()
+    f.set_result(7)
+    f_b = extract_deep([[[[[f]]]]])
+
+    assert f_b.result() == 7
+
+
+@pytest.mark.local
+def test_deeper_list_and_tuple():
+    f = Future()
+    f.set_result(7)
+    f_b = extract_deep([([([f],)],)])
+
+    assert f_b.result() == 7
+
+
+@parsl.python_app
+def dictionary_checker(d):
+    assert d["a"] == [30, 10]
+    assert d["b"] == 20
+
+
+@pytest.mark.local
+def test_dictionary():
+    k1 = Future()
+    k1.set_result("a")
+    k2 = Future()
+    k2.set_result("b")
+    v1 = Future()
+    v1.set_result(10)
+
+    # this .result() will fail if the asserts fail
+    dictionary_checker({k1: [30, v1], k2: 20}).result()
+
+
+@pytest.mark.local
+def test_dictionary_later():
+    k1 = Future()
+    k2 = Future()
+    v1 = Future()
+
+    f1 = dictionary_checker({k1: [30, v1], k2: 20})
+
+    assert not f1.done()
+    k1.set_result("a")
+    k2.set_result("b")
+    v1.set_result(10)
+
+    # having set the results, f1 should fairly rapidly complete (but not
+    # instantly) so we can't assert on done() here... but we can
+    # check that f1 does "eventually" complete... meaning that
+    # none of the assertions inside test_dictionary have failed
+
+    assert f1.result() is None

parsl/tests/test_radical/test_mpi_funcs.py

@@ -1,6 +1,6 @@
-import parsl
 import pytest
 
+import parsl
 from parsl.tests.configs.local_radical_mpi import fresh_config as local_config
 
 

parsl/tests/test_regression/test_1480.py

@@ -1,5 +1,6 @@
-from parsl import python_app
 import pytest
+
+from parsl import python_app
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_regression/test_1653.py

@@ -1,5 +1,6 @@
-from parsl import python_app
 import pytest
+
+from parsl import python_app
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_regression/test_2652.py

@@ -1,4 +1,5 @@
 import pytest
+
 from parsl.jobs.states import JobState
 
 

parsl/tests/test_regression/test_69a.py

@@ -6,7 +6,6 @@ import pytest
 from parsl.app.app import bash_app, python_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/test_regression/test_854.py

@@ -1,8 +1,10 @@
-import time
 import multiprocessing
+import random
+import time
+
 import pytest
+
 from parsl.multiprocessing import MacSafeQueue
-import random
 
 
 def consumer(in_q, out_q, delay=0):

parsl/tests/test_regression/test_97_parallelism_0.py

@@ -1,11 +1,10 @@
 import pytest
 
 import parsl
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-from parsl.providers import LocalProvider
 from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 
 def local_config() -> Config:

parsl/tests/test_regression/test_98.py

@@ -6,7 +6,6 @@ import argparse
 import pytest
 
 import parsl
-
 from parsl.dataflow.dflow import DataFlowKernel
 from parsl.tests.configs.local_threads import config
 

parsl/tests/test_scaling/test_block_error_handler.py

@@ -1,11 +1,16 @@
+from functools import partial
+from unittest.mock import Mock
+
 import pytest
 
 from parsl.executors import HighThroughputExecutor
+from parsl.jobs.error_handlers import (
+    noop_error_handler,
+    simple_error_handler,
+    windowed_error_handler,
+)
+from parsl.jobs.states import JobState, JobStatus
 from parsl.providers import LocalProvider
-from unittest.mock import Mock
-from parsl.jobs.states import JobStatus, JobState
-from parsl.jobs.error_handlers import simple_error_handler, windowed_error_handler, noop_error_handler
-from functools import partial
 
 
 @pytest.mark.local

parsl/tests/test_scaling/test_regression_1621.py

@@ -69,6 +69,4 @@ def test_one_block(tmpd_cwd):
     with parsl.load(config):
         app().result()
 
-    parsl.clear()
-
     assert oneshot_provider.recorded_submits == 1

parsl/tests/test_scaling/test_scale_down.py

@@ -4,13 +4,12 @@ import time
 import pytest
 
 import parsl
-
 from parsl import File, python_app
-from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
 
 logger = logging.getLogger(__name__)