parsl 2024.3.18__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/tests/test_python_apps/test_inputs_default.py

@@ -0,0 +1,22 @@
+import pytest
+
+import parsl
+from parsl import python_app
+from parsl.executors.threads import ThreadPoolExecutor
+
+
+def local_config():
+    return parsl.Config(executors=[ThreadPoolExecutor()])
+
+
+@pytest.mark.local
+def test_default_inputs():
+    @python_app
+    def identity(inp):
+        return inp
+
+    @python_app
+    def add_inputs(inputs=[identity(1), identity(2)]):
+        return sum(inputs)
+
+    assert add_inputs().result() == 3

parsl/tests/test_python_apps/test_join.py

@@ -3,7 +3,6 @@ import pytest
 from parsl import join_app, python_app
 from parsl.dataflow.errors import JoinError
 
-
 RESULT_CONSTANT = 3
 
 

parsl/tests/test_python_apps/test_lifted.py

@@ -1,26 +1,30 @@
+from concurrent.futures import Future
+from typing import TypeVar
+
 import pytest
 
-from concurrent.futures import Future
 from parsl import python_app
 
+T = TypeVar('T')
+
 
 @python_app
-def returns_a_dict():
+def returns_a_dict() -> dict:
     return {"a": "X", "b": "Y"}
 
 
 @python_app
-def returns_a_list():
+def returns_a_list() -> list:
     return ["X", "Y"]
 
 
 @python_app
-def returns_a_tuple():
+def returns_a_tuple() -> tuple:
     return ("X", "Y")
 
 
 @python_app
-def returns_a_class():
+def returns_a_class() -> type:
     from dataclasses import dataclass
 
     @dataclass
@@ -38,7 +42,7 @@ class MyOuterClass():
 
 
 @python_app
-def returns_a_class_instance():
+def returns_a_class_instance() -> object:
     return MyOuterClass()
 
 
@@ -110,7 +114,7 @@ def test_returns_a_class():
 
 
 @python_app
-def passthrough(v):
+def passthrough(v: T) -> T:
     return v
 
 

parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py

@@ -1,4 +1,5 @@
 import pytest
+
 from parsl import python_app
 from parsl.dataflow.memoization import id_for_memo
 

parsl/tests/test_python_apps/test_outputs.py

@@ -16,7 +16,7 @@ def double(x, outputs=[]):
 whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs', '*threads*')
 
 
-@pytest.mark.issue363
+@pytest.mark.shared_fs
 def test_launch_apps(tmpd_cwd, n=2):
     outdir = tmpd_cwd / "outputs"
     outdir.mkdir()

parsl/tests/test_python_apps/test_pluggable_future_resolution.py

@@ -0,0 +1,161 @@
+from concurrent.futures import Future
+from pathlib import Path
+from threading import Event
+from typing import Sequence
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.dataflow.dependency_resolvers import DEEP_DEPENDENCY_RESOLVER
+from parsl.dataflow.errors import DependencyError
+
+
+def local_config():
+    return Config(dependency_resolver=DEEP_DEPENDENCY_RESOLVER)
+
+
+@parsl.python_app
+def a(event):
+    event.wait()
+    return 7
+
+
+@parsl.python_app
+def b(x: int):
+    return x + 1
+
+
+@pytest.mark.local
+def test_simple_pos_arg():
+    e = Event()
+    s = a(e)
+    f_b = b(s)
+    e.set()
+
+    assert f_b.result() == 8
+
+
+@parsl.python_app
+def b_first(x: Sequence[int]):
+    return x[0] + 1
+
+
+@pytest.mark.local
+def test_tuple_pos_arg():
+    e = Event()
+    s = (a(e),)
+    f_b = b_first(s)
+    e.set()
+    assert f_b.result() == 8
+
+
+@pytest.mark.local
+def test_list_exception():
+    a = Future()
+    a.set_exception(RuntimeError("artificial error"))
+    f_b = b([a])
+    assert isinstance(f_b.exception(), DependencyError)
+
+
+@parsl.python_app
+def make_path(s: str):
+    return Path(s)
+
+
+@parsl.python_app
+def append_paths(iterable, end_str: str = "end"):
+    type_ = type(iterable)
+    return type_([Path(s, end_str) for s in iterable])
+
+
+@pytest.mark.local
+@pytest.mark.parametrize(
+    "type_",
+    [
+        tuple,
+        list,
+        set,
+    ],
+)
+def test_resolving_iterables(type_):
+    output1 = make_path("test1")
+    output2 = make_path("test2")
+    output3 = append_paths(type_([output1, output2]), end_str="end")
+    assert output3.result() == type_([Path("test1", "end"), Path("test2", "end")])
+
+
+@parsl.python_app
+def append_paths_dict(iterable: dict, end_str: str = "end"):
+    return {Path(k, end_str): Path(v, end_str) for k, v in iterable.items()}
+
+
+@pytest.mark.local
+def test_resolving_dict():
+    output1 = make_path("test1")
+    output2 = make_path("test2")
+    output3 = append_paths_dict({output1: output2}, end_str="end")
+    assert output3.result() == {Path("test1", "end"): Path("test2", "end")}
+
+
+@parsl.python_app
+def extract_deep(struct: list):
+    return struct[0][0][0][0][0]
+
+
+@pytest.mark.local
+def test_deeper_list():
+    f = Future()
+    f.set_result(7)
+    f_b = extract_deep([[[[[f]]]]])
+
+    assert f_b.result() == 7
+
+
+@pytest.mark.local
+def test_deeper_list_and_tuple():
+    f = Future()
+    f.set_result(7)
+    f_b = extract_deep([([([f],)],)])
+
+    assert f_b.result() == 7
+
+
+@parsl.python_app
+def dictionary_checker(d):
+    assert d["a"] == [30, 10]
+    assert d["b"] == 20
+
+
+@pytest.mark.local
+def test_dictionary():
+    k1 = Future()
+    k1.set_result("a")
+    k2 = Future()
+    k2.set_result("b")
+    v1 = Future()
+    v1.set_result(10)
+
+    # this .result() will fail if the asserts fail
+    dictionary_checker({k1: [30, v1], k2: 20}).result()
+
+
+@pytest.mark.local
+def test_dictionary_later():
+    k1 = Future()
+    k2 = Future()
+    v1 = Future()
+
+    f1 = dictionary_checker({k1: [30, v1], k2: 20})
+
+    assert not f1.done()
+    k1.set_result("a")
+    k2.set_result("b")
+    v1.set_result(10)
+
+    # having set the results, f1 should fairly rapidly complete (but not
+    # instantly) so we can't assert on done() here... but we can
+    # check that f1 does "eventually" complete... meaning that
+    # none of the assertions inside test_dictionary have failed
+
+    assert f1.result() is None

parsl/tests/test_radical/test_mpi_funcs.py

@@ -1,6 +1,6 @@
-import parsl
 import pytest
 
+import parsl
 from parsl.tests.configs.local_radical_mpi import fresh_config as local_config
 
 
@@ -16,7 +16,6 @@ def some_mpi_func(msg, sleep, comm=None, parsl_resource_specification={}):
 apps = []
 
 
-@pytest.mark.skip("hangs in CI - waiting for resolution of issue #3029")
 @pytest.mark.local
 @pytest.mark.radical
 def test_radical_mpi(n=7):

parsl/tests/test_regression/test_1480.py

@@ -1,5 +1,6 @@
-from parsl import python_app
 import pytest
+
+from parsl import python_app
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_regression/test_1653.py

@@ -1,5 +1,6 @@
-from parsl import python_app
 import pytest
+
+from parsl import python_app
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_regression/test_226.py

@@ -53,6 +53,7 @@ def test_get_dataframe():
     assert res.equals(data), 'Unexpected dataframe'
 
 
+@pytest.mark.shared_fs
 def test_bash_default_arg():
     if os.path.exists('std.out'):
         os.remove('std.out')

parsl/tests/test_regression/test_2652.py

@@ -1,4 +1,5 @@
 import pytest
+
 from parsl.jobs.states import JobState
 
 

parsl/tests/test_regression/test_69a.py

@@ -6,7 +6,6 @@ import pytest
 from parsl.app.app import bash_app, python_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/test_regression/test_854.py

@@ -1,8 +1,10 @@
-import time
 import multiprocessing
+import random
+import time
+
 import pytest
+
 from parsl.multiprocessing import MacSafeQueue
-import random
 
 
 def consumer(in_q, out_q, delay=0):

parsl/tests/test_regression/test_97_parallelism_0.py

@@ -1,11 +1,10 @@
 import pytest
 
 import parsl
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-from parsl.providers import LocalProvider
 from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 
 def local_config() -> Config:

parsl/tests/test_regression/test_98.py

@@ -6,7 +6,6 @@ import argparse
 import pytest
 
 import parsl
-
 from parsl.dataflow.dflow import DataFlowKernel
 from parsl.tests.configs.local_threads import config
 

parsl/tests/test_scaling/test_block_error_handler.py

@@ -1,11 +1,16 @@
+from functools import partial
+from unittest.mock import Mock
+
 import pytest
 
 from parsl.executors import HighThroughputExecutor
+from parsl.jobs.error_handlers import (
+    noop_error_handler,
+    simple_error_handler,
+    windowed_error_handler,
+)
+from parsl.jobs.states import JobState, JobStatus
 from parsl.providers import LocalProvider
-from unittest.mock import Mock
-from parsl.jobs.states import JobStatus, JobState
-from parsl.jobs.error_handlers import simple_error_handler, windowed_error_handler, noop_error_handler
-from functools import partial
 
 
 @pytest.mark.local

parsl/tests/test_scaling/test_regression_1621.py

@@ -3,12 +3,19 @@ import threading
 import pytest
 
 import parsl
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
 from parsl.providers import LocalProvider
 
+# Timing notes:
+# The configured strategy_period must be much smaller than the delay in
+# app() so that multiple iterations of the strategy have had a chance
+# to (mis)behave.
+# The status polling interval in OneShotLocalProvider must be much bigger
+# than the above times, so that the job status cached from the provider
+# will not be updated while the single invocation of app() runs.
+
 
 @parsl.python_app
 def app():
@@ -35,7 +42,6 @@ def test_one_block(tmpd_cwd):
     one app is invoked. this is a regression test.
     """
     oneshot_provider = OneShotLocalProvider(
-        channel=LocalChannel(),
         init_blocks=0,
         min_blocks=0,
         max_blocks=10,
@@ -55,20 +61,10 @@ def test_one_block(tmpd_cwd):
             )
         ],
         strategy='simple',
+        strategy_period=0.1
     )
 
-    parsl.load(config)
-    dfk = parsl.dfk()
-
-    def poller():
-        import time
-        while True:
-            dfk.job_status_poller.poll()
-            time.sleep(0.1)
-
-    threading.Thread(target=poller, daemon=True).start()
-    app().result()
-    parsl.dfk().cleanup()
-    parsl.clear()
+    with parsl.load(config):
+        app().result()
 
     assert oneshot_provider.recorded_submits == 1

parsl/tests/test_scaling/test_regression_3568_scaledown_vs_MISSING.py

@@ -0,0 +1,84 @@
+import time
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import WrappedLauncher
+from parsl.providers import LocalProvider
+
+
+def local_config():
+    # see the comments inside test_regression for reasoning about why each
+    # of these parameters is set why it is.
+    return Config(
+        max_idletime=1,
+
+        strategy='htex_auto_scale',
+        strategy_period=1,
+
+        executors=[
+            HighThroughputExecutor(
+                label="htex_local",
+                encrypted=True,
+                provider=LocalProvider(
+                    init_blocks=1,
+                    min_blocks=0,
+                    max_blocks=1,
+                    launcher=WrappedLauncher(prepend="sleep inf ; "),
+                ),
+            )
+        ],
+    )
+
+
+@parsl.python_app
+def task():
+    return 7
+
+
+@pytest.mark.local
+def test_regression(try_assert):
+    # The above config means that we should start scaling out one initial
+    # block, but then scale it back in after a second or so if the executor
+    # is kept idle (which this test does using try_assert).
+
+    # Because of 'sleep inf' in the WrappedLaucher, the block will not ever
+    # register.
+
+    # The bug being tested is about mistreatment of blocks which are scaled in
+    # before they have a chance to register, and the above forces that to
+    # happen.
+
+    # After that scaling in has happened, we should see that we have one block
+    # and it should be in a terminal state. The below try_assert waits for
+    # that to become true.
+
+    # At that time, we should also see htex reporting no blocks registered - as
+    # mentioned above, that is a necessary part of the bug being tested here.
+
+    # Give 10 strategy periods for the above to happen: each step of scale up,
+    # and scale down due to idleness isn't guaranteed to happen in exactly one
+    # scaling step.
+
+    htex = parsl.dfk().executors['htex_local']
+
+    try_assert(lambda: len(htex.status_facade) == 1 and htex.status_facade['0'].terminal,
+               timeout_ms=10000)
+
+    assert htex.connected_blocks() == [], "No block should have connected to interchange"
+
+    # Now we can reconfigure the launcher to let subsequent blocks launch ok,
+    # and run a trivial task. That trivial task will scale up a new block and
+    # run the task successfully.
+
+    # Prior to issue #3568, the bug was that the scale in of the first
+    # block earlier in the test case would have incorrectly been treated as a
+    # failure, and then the block error handler would have treated that failure
+    # as a permanent htex failure, and so the task execution below would raise
+    # a BadStateException rather than attempt to run the task.
+
+    assert htex.provider.launcher.prepend != "", "Pre-req: prepend attribute should exist and be non-empty"
+    htex.provider.launcher.prepend = ""
+    assert task().result() == 7

parsl/tests/test_scaling/test_regression_3696_oscillation.py

@@ -0,0 +1,103 @@
+import math
+from unittest.mock import MagicMock
+
+import pytest
+
+from parsl.executors.high_throughput.executor import HighThroughputExecutor
+from parsl.jobs.states import JobState, JobStatus
+from parsl.jobs.strategy import Strategy
+
+
+# the parameterize tuple consists of:
+# Input:
+#  * number of tasks to mock the load as
+#  * number of workers per node
+# Expected output:
+#  * the number of blocks we should expect to be launched
+#    in this situation
+#
+# This test will configure an executor, then run strategize
+# a few times, asserting that it converges to the correct
+# number of blocks without oscillating.
+@pytest.mark.local
+@pytest.mark.parametrize("ns", [(14, 48, 1),    # values from issue #3696
+
+                                (1, 1, 1),      # one task needs one block
+
+                                (100, 1, 20),   # many one-task blocks, hitting hard-coded max blocks
+
+                                (47, 48, 1),    # some edge cases around #3696 values
+                                (48, 48, 1),    # "
+                                (49, 48, 2),    # "
+                                (149, 50, 3)])  # "
+def test_htex_strategy_does_not_oscillate(ns):
+    """Check for oscillations in htex scaling.
+    In issue 3696, with a large number of workers per block
+    and a smaller number of active tasks, the htex scaling
+    strategy oscillates between 0 and 1 active block, rather
+    than converging to 1 active block.
+    """
+
+    n_tasks, n_workers, n_blocks = ns
+
+    s = Strategy(strategy='htex_auto_scale', max_idletime=0)
+
+    provider = MagicMock()
+    executor = MagicMock(spec=HighThroughputExecutor)
+
+    statuses = {}
+
+    executor.provider = provider
+    executor.outstanding = n_tasks
+    executor.status_facade = statuses
+    executor.workers_per_node = n_workers
+
+    provider.parallelism = 1
+    provider.init_blocks = 0
+    provider.min_blocks = 0
+    provider.max_blocks = 20
+    provider.nodes_per_block = 1
+
+    def scale_out(n):
+        for _ in range(n):
+            statuses[len(statuses)] = JobStatus(state=JobState.PENDING)
+
+    executor.scale_out_facade.side_effect = scale_out
+
+    def scale_in(n, max_idletime=None):
+        # find n PENDING jobs and set them to CANCELLED
+        for k in statuses:
+            if n == 0:
+                return
+            if statuses[k].state == JobState.PENDING:
+                statuses[k].state = JobState.CANCELLED
+                n -= 1
+
+    executor.scale_in_facade.side_effect = scale_in
+
+    s.add_executors([executor])
+
+    # In issue #3696, this first strategise does initial and load based
+    # scale outs, because n_tasks > n_workers*0
+    s.strategize([executor])
+
+    executor.scale_out_facade.assert_called()
+    assert len(statuses) == n_blocks, "Should have launched n_blocks"
+    assert len([k for k in statuses if statuses[k].state == JobState.PENDING]) == n_blocks
+    # there might be several calls to scale_out_facade inside strategy,
+    # but the end effect should be that exactly one block is scaled out.
+
+    executor.scale_in_facade.assert_not_called()
+
+    # In issue #3696, this second strategize does a scale in, because n_tasks < n_workers*1
+    s.strategize([executor])
+
+    # assert that there should still be n_blocks pending blocks
+    assert len([k for k in statuses if statuses[k].state == JobState.PENDING]) == n_blocks
+    # this assert fails due to issue #3696
+
+    # Now check scale in happens with 0 load
+    executor.outstanding = 0
+    s.strategize([executor])
+    executor.scale_in_facade.assert_called()
+    assert len([k for k in statuses if statuses[k].state == JobState.PENDING]) == 0

parsl/tests/test_scaling/test_scale_down.py

@@ -4,13 +4,11 @@ import time
 import pytest
 
 import parsl
-
 from parsl import File, python_app
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
 
 logger = logging.getLogger(__name__)
 
@@ -30,7 +28,6 @@ def local_config():
                 max_workers_per_node=1,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=0,
                     max_blocks=_max_blocks,
                     min_blocks=_min_blocks,

parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py

@@ -1,15 +1,13 @@
+from threading import Event
+
 import pytest
 
 import parsl
-
 from parsl import File, python_app
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-
-from threading import Event
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
 
 _max_blocks = 5
 _min_blocks = 0
@@ -24,10 +22,9 @@ def local_config():
                 poll_period=100,
                 label="htex_local",
                 address="127.0.0.1",
-                max_workers=1,
+                max_workers_per_node=1,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=0,
                     max_blocks=_max_blocks,
                     min_blocks=_min_blocks,