parsl 2024.3.11__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/tests/test_python_apps/test_dependencies_deep.py

@@ -0,0 +1,59 @@
+import inspect
+from concurrent.futures import Future
+from typing import Any, Callable, Dict
+
+import pytest
+
+import parsl
+from parsl.executors.base import ParslExecutor
+
+# N is the number of tasks to chain
+# With mid-2024 Parsl, N>140 causes Parsl to hang
+N = 100
+
+# MAX_STACK is the maximum Python stack depth allowed for either
+# task submission to an executor or execution of a task.
+# With mid-2024 Parsl, 2-3 stack entries will be used per
+# recursively launched parsl task. So this should be smaller than
+# 2*N, but big enough to allow regular pytest+parsl stuff to
+# happen.
+MAX_STACK = 50
+
+
+def local_config():
+    return parsl.Config(executors=[ImmediateExecutor()])
+
+
+class ImmediateExecutor(ParslExecutor):
+    def start(self):
+        pass
+
+    def shutdown(self):
+        pass
+
+    def submit(self, func: Callable, resource_specification: Dict[str, Any], *args: Any, **kwargs: Any) -> Future:
+        stack_depth = len(inspect.stack())
+        assert stack_depth < MAX_STACK, "tasks should not be launched deep in the Python stack"
+        fut: Future[None] = Future()
+        res = func(*args, **kwargs)
+        fut.set_result(res)
+        return fut
+
+
+@parsl.python_app
+def chain(upstream):
+    stack_depth = len(inspect.stack())
+    assert stack_depth < MAX_STACK, "chained dependencies should not be launched deep in the Python stack"
+
+
+@pytest.mark.local
+def test_deep_dependency_stack_depth():
+
+    fut = Future()
+    here = fut
+
+    for _ in range(N):
+        here = chain(here)
+
+    fut.set_result(None)
+    here.result()

parsl/tests/test_python_apps/test_fail.py

@@ -41,28 +41,3 @@ def test_fail_sequence(fail_probs):
 
     with pytest.raises(DependencyError):
         t_final.result()
-
-
-def test_deps(width=3):
-    """Random failures in branches of Map -> Map -> reduce"""
-    # App1   App2  ... AppN
-    futs = [random_fail(fail_prob=0.4) for _ in range(width)]
-
-    # App1   App2  ... AppN
-    # |       |        |
-    # V       V        V
-    # App1   App2  ... AppN
-
-    futs = [random_fail(fail_prob=0.8, inputs=[f]) for f in futs]
-
-    # App1   App2  ... AppN
-    #   |       |        |
-    #   V       V        V
-    # App1   App2  ... AppN
-    #    \      |       /
-    #     \     |      /
-    # App_Final
-    try:
-        random_fail(fail_prob=0, inputs=futs).result()
-    except DependencyError:
-        pass

parsl/tests/test_python_apps/test_futures.py

@@ -11,9 +11,10 @@ Same applies to datafutures, and we need to know the behavior wrt.
 2. done() called on 1, vs 2
 
 """
-import pytest
 from os.path import basename
 
+import pytest
+
 from parsl.app.app import python_app
 from parsl.data_provider.files import File
 

parsl/tests/test_python_apps/test_inputs_default.py

@@ -0,0 +1,22 @@
+import pytest
+
+import parsl
+from parsl import python_app
+from parsl.executors.threads import ThreadPoolExecutor
+
+
+def local_config():
+    return parsl.Config(executors=[ThreadPoolExecutor()])
+
+
+@pytest.mark.local
+def test_default_inputs():
+    @python_app
+    def identity(inp):
+        return inp
+
+    @python_app
+    def add_inputs(inputs=[identity(1), identity(2)]):
+        return sum(inputs)
+
+    assert add_inputs().result() == 3

parsl/tests/test_python_apps/test_join.py

@@ -3,7 +3,6 @@ import pytest
 from parsl import join_app, python_app
 from parsl.dataflow.errors import JoinError
 
-
 RESULT_CONSTANT = 3
 
 

parsl/tests/test_python_apps/test_lifted.py

@@ -1,26 +1,30 @@
+from concurrent.futures import Future
+from typing import TypeVar
+
 import pytest
 
-from concurrent.futures import Future
 from parsl import python_app
 
+T = TypeVar('T')
+
 
 @python_app
-def returns_a_dict():
+def returns_a_dict() -> dict:
     return {"a": "X", "b": "Y"}
 
 
 @python_app
-def returns_a_list():
+def returns_a_list() -> list:
     return ["X", "Y"]
 
 
 @python_app
-def returns_a_tuple():
+def returns_a_tuple() -> tuple:
     return ("X", "Y")
 
 
 @python_app
-def returns_a_class():
+def returns_a_class() -> type:
     from dataclasses import dataclass
 
     @dataclass
@@ -38,7 +42,7 @@ class MyOuterClass():
 
 
 @python_app
-def returns_a_class_instance():
+def returns_a_class_instance() -> object:
     return MyOuterClass()
 
 
@@ -110,7 +114,7 @@ def test_returns_a_class():
 
 
 @python_app
-def passthrough(v):
+def passthrough(v: T) -> T:
     return v
 
 

parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py

@@ -1,4 +1,5 @@
 import pytest
+
 from parsl import python_app
 from parsl.dataflow.memoization import id_for_memo
 

parsl/tests/test_python_apps/test_outputs.py

@@ -16,7 +16,7 @@ def double(x, outputs=[]):
 whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs', '*threads*')
 
 
-@pytest.mark.issue363
+@pytest.mark.shared_fs
 def test_launch_apps(tmpd_cwd, n=2):
     outdir = tmpd_cwd / "outputs"
     outdir.mkdir()

parsl/tests/test_python_apps/test_pluggable_future_resolution.py

@@ -0,0 +1,161 @@
+from concurrent.futures import Future
+from pathlib import Path
+from threading import Event
+from typing import Sequence
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.dataflow.dependency_resolvers import DEEP_DEPENDENCY_RESOLVER
+from parsl.dataflow.errors import DependencyError
+
+
+def local_config():
+    return Config(dependency_resolver=DEEP_DEPENDENCY_RESOLVER)
+
+
+@parsl.python_app
+def a(event):
+    event.wait()
+    return 7
+
+
+@parsl.python_app
+def b(x: int):
+    return x + 1
+
+
+@pytest.mark.local
+def test_simple_pos_arg():
+    e = Event()
+    s = a(e)
+    f_b = b(s)
+    e.set()
+
+    assert f_b.result() == 8
+
+
+@parsl.python_app
+def b_first(x: Sequence[int]):
+    return x[0] + 1
+
+
+@pytest.mark.local
+def test_tuple_pos_arg():
+    e = Event()
+    s = (a(e),)
+    f_b = b_first(s)
+    e.set()
+    assert f_b.result() == 8
+
+
+@pytest.mark.local
+def test_list_exception():
+    a = Future()
+    a.set_exception(RuntimeError("artificial error"))
+    f_b = b([a])
+    assert isinstance(f_b.exception(), DependencyError)
+
+
+@parsl.python_app
+def make_path(s: str):
+    return Path(s)
+
+
+@parsl.python_app
+def append_paths(iterable, end_str: str = "end"):
+    type_ = type(iterable)
+    return type_([Path(s, end_str) for s in iterable])
+
+
+@pytest.mark.local
+@pytest.mark.parametrize(
+    "type_",
+    [
+        tuple,
+        list,
+        set,
+    ],
+)
+def test_resolving_iterables(type_):
+    output1 = make_path("test1")
+    output2 = make_path("test2")
+    output3 = append_paths(type_([output1, output2]), end_str="end")
+    assert output3.result() == type_([Path("test1", "end"), Path("test2", "end")])
+
+
+@parsl.python_app
+def append_paths_dict(iterable: dict, end_str: str = "end"):
+    return {Path(k, end_str): Path(v, end_str) for k, v in iterable.items()}
+
+
+@pytest.mark.local
+def test_resolving_dict():
+    output1 = make_path("test1")
+    output2 = make_path("test2")
+    output3 = append_paths_dict({output1: output2}, end_str="end")
+    assert output3.result() == {Path("test1", "end"): Path("test2", "end")}
+
+
+@parsl.python_app
+def extract_deep(struct: list):
+    return struct[0][0][0][0][0]
+
+
+@pytest.mark.local
+def test_deeper_list():
+    f = Future()
+    f.set_result(7)
+    f_b = extract_deep([[[[[f]]]]])
+
+    assert f_b.result() == 7
+
+
+@pytest.mark.local
+def test_deeper_list_and_tuple():
+    f = Future()
+    f.set_result(7)
+    f_b = extract_deep([([([f],)],)])
+
+    assert f_b.result() == 7
+
+
+@parsl.python_app
+def dictionary_checker(d):
+    assert d["a"] == [30, 10]
+    assert d["b"] == 20
+
+
+@pytest.mark.local
+def test_dictionary():
+    k1 = Future()
+    k1.set_result("a")
+    k2 = Future()
+    k2.set_result("b")
+    v1 = Future()
+    v1.set_result(10)
+
+    # this .result() will fail if the asserts fail
+    dictionary_checker({k1: [30, v1], k2: 20}).result()
+
+
+@pytest.mark.local
+def test_dictionary_later():
+    k1 = Future()
+    k2 = Future()
+    v1 = Future()
+
+    f1 = dictionary_checker({k1: [30, v1], k2: 20})
+
+    assert not f1.done()
+    k1.set_result("a")
+    k2.set_result("b")
+    v1.set_result(10)
+
+    # having set the results, f1 should fairly rapidly complete (but not
+    # instantly) so we can't assert on done() here... but we can
+    # check that f1 does "eventually" complete... meaning that
+    # none of the assertions inside test_dictionary have failed
+
+    assert f1.result() is None

parsl/tests/test_radical/test_mpi_funcs.py

@@ -1,6 +1,6 @@
-import parsl
 import pytest
 
+import parsl
 from parsl.tests.configs.local_radical_mpi import fresh_config as local_config
 
 
@@ -16,7 +16,6 @@ def some_mpi_func(msg, sleep, comm=None, parsl_resource_specification={}):
 apps = []
 
 
-@pytest.mark.skip("hangs in CI - waiting for resolution of issue #3029")
 @pytest.mark.local
 @pytest.mark.radical
 def test_radical_mpi(n=7):

parsl/tests/test_regression/test_1480.py

@@ -1,5 +1,6 @@
-from parsl import python_app
 import pytest
+
+from parsl import python_app
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_regression/test_1653.py

@@ -1,5 +1,6 @@
-from parsl import python_app
 import pytest
+
+from parsl import python_app
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_regression/test_226.py

@@ -53,6 +53,7 @@ def test_get_dataframe():
     assert res.equals(data), 'Unexpected dataframe'
 
 
+@pytest.mark.shared_fs
 def test_bash_default_arg():
     if os.path.exists('std.out'):
         os.remove('std.out')

parsl/tests/test_regression/test_2652.py

@@ -1,4 +1,5 @@
 import pytest
+
 from parsl.jobs.states import JobState
 
 

parsl/tests/test_regression/test_69a.py

@@ -6,7 +6,6 @@ import pytest
 from parsl.app.app import bash_app, python_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/test_regression/test_854.py

@@ -1,8 +1,10 @@
-import time
 import multiprocessing
+import random
+import time
+
 import pytest
+
 from parsl.multiprocessing import MacSafeQueue
-import random
 
 
 def consumer(in_q, out_q, delay=0):

parsl/tests/test_regression/test_97_parallelism_0.py

@@ -1,11 +1,10 @@
 import pytest
 
 import parsl
-
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-from parsl.providers import LocalProvider
 from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 
 
 def local_config() -> Config:

parsl/tests/test_regression/test_98.py

@@ -6,7 +6,6 @@ import argparse
 import pytest
 
 import parsl
-
 from parsl.dataflow.dflow import DataFlowKernel
 from parsl.tests.configs.local_threads import config
 

parsl/tests/test_scaling/test_block_error_handler.py

@@ -1,11 +1,16 @@
+from functools import partial
+from unittest.mock import Mock
+
 import pytest
 
 from parsl.executors import HighThroughputExecutor
+from parsl.jobs.error_handlers import (
+    noop_error_handler,
+    simple_error_handler,
+    windowed_error_handler,
+)
+from parsl.jobs.states import JobState, JobStatus
 from parsl.providers import LocalProvider
-from unittest.mock import Mock
-from parsl.jobs.states import JobStatus, JobState
-from parsl.jobs.error_handlers import simple_error_handler, windowed_error_handler, noop_error_handler
-from functools import partial
 
 
 @pytest.mark.local

parsl/tests/test_scaling/test_regression_1621.py

@@ -3,12 +3,19 @@ import threading
 import pytest
 
 import parsl
-from parsl.channels import LocalChannel
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
 from parsl.providers import LocalProvider
 
+# Timing notes:
+# The configured strategy_period must be much smaller than the delay in
+# app() so that multiple iterations of the strategy have had a chance
+# to (mis)behave.
+# The status polling interval in OneShotLocalProvider must be much bigger
+# than the above times, so that the job status cached from the provider
+# will not be updated while the single invocation of app() runs.
+
 
 @parsl.python_app
 def app():
@@ -35,7 +42,6 @@ def test_one_block(tmpd_cwd):
     one app is invoked. this is a regression test.
     """
     oneshot_provider = OneShotLocalProvider(
-        channel=LocalChannel(),
         init_blocks=0,
         min_blocks=0,
         max_blocks=10,
@@ -55,20 +61,10 @@ def test_one_block(tmpd_cwd):
             )
         ],
         strategy='simple',
+        strategy_period=0.1
     )
 
-    parsl.load(config)
-    dfk = parsl.dfk()
-
-    def poller():
-        import time
-        while True:
-            dfk.job_status_poller.poll()
-            time.sleep(0.1)
-
-    threading.Thread(target=poller, daemon=True).start()
-    app().result()
-    parsl.dfk().cleanup()
-    parsl.clear()
+    with parsl.load(config):
+        app().result()
 
     assert oneshot_provider.recorded_submits == 1

parsl/tests/test_scaling/test_regression_3568_scaledown_vs_MISSING.py

@@ -0,0 +1,84 @@
+import time
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import WrappedLauncher
+from parsl.providers import LocalProvider
+
+
+def local_config():
+    # see the comments inside test_regression for reasoning about why each
+    # of these parameters is set why it is.
+    return Config(
+        max_idletime=1,
+
+        strategy='htex_auto_scale',
+        strategy_period=1,
+
+        executors=[
+            HighThroughputExecutor(
+                label="htex_local",
+                encrypted=True,
+                provider=LocalProvider(
+                    init_blocks=1,
+                    min_blocks=0,
+                    max_blocks=1,
+                    launcher=WrappedLauncher(prepend="sleep inf ; "),
+                ),
+            )
+        ],
+    )
+
+
+@parsl.python_app
+def task():
+    return 7
+
+
+@pytest.mark.local
+def test_regression(try_assert):
+    # The above config means that we should start scaling out one initial
+    # block, but then scale it back in after a second or so if the executor
+    # is kept idle (which this test does using try_assert).
+
+    # Because of 'sleep inf' in the WrappedLaucher, the block will not ever
+    # register.
+
+    # The bug being tested is about mistreatment of blocks which are scaled in
+    # before they have a chance to register, and the above forces that to
+    # happen.
+
+    # After that scaling in has happened, we should see that we have one block
+    # and it should be in a terminal state. The below try_assert waits for
+    # that to become true.
+
+    # At that time, we should also see htex reporting no blocks registered - as
+    # mentioned above, that is a necessary part of the bug being tested here.
+
+    # Give 10 strategy periods for the above to happen: each step of scale up,
+    # and scale down due to idleness isn't guaranteed to happen in exactly one
+    # scaling step.
+
+    htex = parsl.dfk().executors['htex_local']
+
+    try_assert(lambda: len(htex.status_facade) == 1 and htex.status_facade['0'].terminal,
+               timeout_ms=10000)
+
+    assert htex.connected_blocks() == [], "No block should have connected to interchange"
+
+    # Now we can reconfigure the launcher to let subsequent blocks launch ok,
+    # and run a trivial task. That trivial task will scale up a new block and
+    # run the task successfully.
+
+    # Prior to issue #3568, the bug was that the scale in of the first
+    # block earlier in the test case would have incorrectly been treated as a
+    # failure, and then the block error handler would have treated that failure
+    # as a permanent htex failure, and so the task execution below would raise
+    # a BadStateException rather than attempt to run the task.
+
+    assert htex.provider.launcher.prepend != "", "Pre-req: prepend attribute should exist and be non-empty"
+    htex.provider.launcher.prepend = ""
+    assert task().result() == 7