parsl 2023.7.3__py3-none-any.whl → 2023.7.17__py3-none-any.whl

parsl/tests/test_data/test_output_chain_filenames.py

@@ -1,46 +1,36 @@
-import argparse
-import os
-
-import pytest
-
-import parsl
-
 from concurrent.futures import Future
+
 from parsl import File
 from parsl.app.app import bash_app
 
 
 @bash_app
-def app1(inputs=[], outputs=[], stdout=None, stderr=None, mock=False):
-    cmd_line = f"""echo 'test' > {outputs[0]}"""
-    return cmd_line
+def app1(inputs=(), outputs=(), stdout=None, stderr=None, mock=False):
+    return f"echo 'test' > {outputs[0]}"
 
 
 @bash_app
-def app2(inputs=[], outputs=[], stdout=None, stderr=None, mock=False):
-
-    with open('somefile.txt', 'w') as f:
-        f.write("%s\n" % inputs[0])
-    cmd_line = f"""echo '{inputs[0]}' > {outputs[0]}"""
-    return cmd_line
+def app2(inputs=(), outputs=(), stdout=None, stderr=None, mock=False):
+    return f"echo '{inputs[0]}' > {outputs[0]}"
 
 
-def test_behavior():
-    app1_future = app1(inputs=[],
-                       outputs=[File("simple-out.txt")])
+def test_behavior(tmpd_cwd):
+    expected_path = str(tmpd_cwd / "simple-out.txt")
+    app1_future = app1(
+        inputs=[],
+        outputs=[File(expected_path)]
+    )
 
     o = app1_future.outputs[0]
     assert isinstance(o, Future)
 
-    app2_future = app2(inputs=[o],
-                       outputs=[File("simple-out2.txt")])
+    app2_future = app2(
+        inputs=[o],
+        outputs=[File(str(tmpd_cwd / "simple-out2.txt"))]
+    )
     app2_future.result()
 
-    expected_name = 'b'
-    with open('somefile.txt', 'r') as f:
-        name = f.read()
-
     with open(app2_future.outputs[0].filepath, 'r') as f:
-        expected_name = f.read()
+        name = f.read().strip()
 
-    assert name == expected_name, "Filename mangled due to DataFuture handling"
+    assert name == expected_path, "Filename mangled due to DataFuture handling"

parsl/tests/test_docs/test_workflow4.py

@@ -1,53 +1,43 @@
-import os
-import parsl
+import pytest
 
 from parsl.app.app import bash_app, python_app
-from parsl.tests.configs.local_threads import config
 from parsl.data_provider.files import File
 
-import pytest
-
-# parsl.set_stream_logger()
-
 
 @bash_app
-def generate(outputs=[]):
-    return "echo $(( RANDOM % (10 - 5 + 1 ) + 5 )) &> {o}".format(o=outputs[0])
+def generate(outputs=()):
+    return "echo 1 &> {o}".format(o=outputs[0])
 
 
 @bash_app
-def concat(inputs=[], outputs=[], stdout="stdout.txt", stderr='stderr.txt'):
+def concat(inputs=(), outputs=(), stdout=None, stderr=None):
     return "cat {0} >> {1}".format(" ".join(map(lambda x: x.filepath, inputs)), outputs[0])
 
 
 @python_app
-def total(inputs=[]):
-    total = 0
-    with open(inputs[0].filepath, 'r') as f:
-        for line in f:
-            total += int(line)
-    return total
+def total(inputs=()):
+    with open(inputs[0].filepath, "r") as f:
+        return sum(int(line) for line in f)
 
 
 @pytest.mark.staging_required
-def test_parallel_dataflow():
+@pytest.mark.parametrize("width", (5, 10, 15))
+def test_parallel_dataflow(tmpd_cwd, width):
     """Test parallel dataflow from docs on Composing workflows
     """
 
-    if os.path.exists('all.txt'):
-        os.remove('all.txt')
-
     # create 5 files with random numbers
-    output_files = []
-    for i in range(5):
-        if os.path.exists('random-%s.txt' % i):
-            os.remove('random-%s.txt' % i)
-        output_files.append(generate(outputs=[File('random-%s.txt' % i)]))
+    output_files = [
+        generate(outputs=[File(str(tmpd_cwd / f"random-{i}.txt"))])
+        for i in range(width)
+    ]
 
     # concatenate the files into a single file
-    cc = concat(inputs=[i.outputs[0]
-                        for i in output_files], outputs=[File("all.txt")])
+    cc = concat(
+        inputs=[i.outputs[0] for i in output_files],
+        outputs=[File(str(tmpd_cwd / "all.txt"))]
+    )
 
     # calculate the average of the random numbers
     totals = total(inputs=[cc.outputs[0]])
-    print(totals.result())
+    assert totals.result() == len(output_files)

parsl/tests/test_error_handling/test_htex_worker_failure.py

@@ -1,28 +1,21 @@
 import pytest
 
-import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.htex_local import fresh_config
-
 from parsl.executors.high_throughput.errors import WorkerLost
 
 
-def local_setup():
+def local_config():
+    from parsl.tests.configs.htex_local import fresh_config
     config = fresh_config()
     config.executors[0].poll_period = 1
     config.executors[0].max_workers = 1
-    parsl.load(config)
-
-
-def local_teardown():
-    parsl.dfk().cleanup()
-    parsl.clear()
+    config.executors[0].heartbeat_period = 1
+    return config
 
 
 @python_app
 def kill_worker():
-    import sys
-    sys.exit(2)
+    raise SystemExit(2)
 
 
 @pytest.mark.local

parsl/tests/test_python_apps/test_fail.py

@@ -1,89 +1,59 @@
-import argparse
+import pytest
 
-import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.local_threads import fresh_config as local_config
+from parsl.dataflow.errors import DependencyError
 
 
-@python_app
-def sleep_fail(sleep_dur, sleep_rand_max, fail_prob, inputs=[]):
-    import time
-    import random
+class ManufacturedTestFailure(Exception):
+    pass
 
-    s = sleep_dur + random.randint(-sleep_rand_max, sleep_rand_max)
 
-    time.sleep(s)
-    raise Exception("App failure")
+@python_app
+def random_fail(fail_prob: float, inputs=()):
+    import random
+    if random.random() < fail_prob:
+        raise ManufacturedTestFailure("App failure")
 
 
-def test_no_deps(numtasks=2):
+def test_no_deps():
     """Test basic error handling, with no dependent failures
     """
+    futs = [random_fail(1), random_fail(0), random_fail(0)]
 
-    fus = []
-    for i in range(0, numtasks):
-
-        fu = sleep_fail(0.1, 0, .8)
-        fus.extend([fu])
-
-    count = 0
-    for fu in fus:
+    for f in futs:
         try:
-            fu.result()
-        except Exception as e:
-            print("Caught exception : ", "*" * 20)
-            print(e)
-            print("*" * 20)
-            count += 1
+            f.result()
+        except ManufacturedTestFailure:
+            pass
 
-    print("Caught failures of  {0}/{1}".format(count, len(fus)))
 
-
-def test_fail_sequence(numtasks=2):
+@pytest.mark.parametrize("fail_probs", ((1, 0), (0, 1)))
+def test_fail_sequence(fail_probs):
     """Test failure in a sequence of dependencies
 
     App1 -> App2 ... -> AppN
     """
 
-    sleep_dur = 0.1
-    fail_prob = 0.4
-
-    fus = {0: None}
-    for i in range(0, numtasks):
-        print("Chaining {0} to {1}".format(i + 1, fus[i]))
-        fus[i + 1] = sleep_fail(sleep_dur, 0, fail_prob, inputs=[fus[i]])
+    t1_fail_prob, t2_fail_prob = fail_probs
+    t1 = random_fail(fail_prob=t1_fail_prob)
+    t2 = random_fail(fail_prob=t2_fail_prob, inputs=[t1])
+    t_final = random_fail(fail_prob=0, inputs=[t2])
 
-    # time.sleep(numtasks*sleep_dur)
-    for k in sorted(fus.keys()):
-        try:
-            x = fus[i].result()
-            print("{0} : {1}".format(k, x))
-        except Exception as e:
-            print("{0} : {1}".format(k, e))
-
-    return
-
-
-def test_deps(numtasks=2):
-    """Random failures in branches of Map -> Map -> reduce
+    with pytest.raises(DependencyError):
+        t_final.result()
 
-    App1   App2  ... AppN
-    """
 
-    fus = []
-    for i in range(0, numtasks):
-        fu = sleep_fail(0.2, 0, .4)
-        fus.extend([fu])
+def test_deps(width=3):
+    """Random failures in branches of Map -> Map -> reduce"""
+    # App1   App2  ... AppN
+    futs = [random_fail(fail_prob=0.4) for _ in range(width)]
 
     # App1   App2  ... AppN
     # |       |        |
     # V       V        V
     # App1   App2  ... AppN
 
-    fus_2 = []
-    for fu in fus:
-        fu = sleep_fail(0, 0, .8, inputs=[fu])
-        fus_2.extend([fu])
+    futs = [random_fail(fail_prob=0.8, inputs=[f]) for f in futs]
 
     # App1   App2  ... AppN
     #   |       |        |
@@ -92,15 +62,7 @@ def test_deps(numtasks=2):
     #    \      |       /
     #     \     |      /
     # App_Final
-
-    fu_final = sleep_fail(1, 0, 0, inputs=fus_2)
-
     try:
-        print("Final status : ", fu_final.result())
-    except parsl.dataflow.errors.DependencyError as e:
-        print("Caught the right exception")
-        print("Exception : ", e)
-    except Exception as e:
-        assert False, "Expected DependencyError but got: %s" % e
-    else:
-        raise RuntimeError("Expected DependencyError, but got no exception")
+        random_fail(fail_prob=0, inputs=futs).result()
+    except DependencyError:
+        pass

parsl/tests/test_python_apps/test_garbage_collect.py

@@ -1,30 +1,36 @@
-import parsl
+import threading
 import time
 
+import pytest
+
+import parsl
 from parsl.app.app import python_app
+from parsl.tests.configs.local_threads import fresh_config as local_config  # noqa
 
 
 @python_app
-def slow_double(x):
-    import time
-    time.sleep(0.1)
+def slow_double(x, may_continue: threading.Event):
+    may_continue.wait()
     return x * 2
 
 
+@pytest.mark.local
 def test_garbage_collect():
     """ Launches an app with a dependency and waits till it's done and asserts that
     the internal refs were wiped
     """
-    x = slow_double(slow_double(10))
+    evt = threading.Event()
+    x = slow_double(10, evt)
+    x = slow_double(x, evt)
 
-    if x.done() is False:
-        assert parsl.dfk().tasks[x.tid]['app_fu'] == x, "Tasks table should have app_fu ref before done"
+    assert parsl.dfk().tasks[x.tid]['app_fu'] == x, "Tasks table should have app_fu ref before done"
 
-    x.result()
+    evt.set()
+    assert x.result() == 10 * 4
     if parsl.dfk().checkpoint_mode is not None:
         # We explicit call checkpoint if checkpoint_mode is enabled covering
         # cases like manual/periodic where checkpointing may be deferred.
         parsl.dfk().checkpoint()
 
-    time.sleep(0.2)  # Give enough time for task wipes to work
+    time.sleep(0.01)  # Give enough time for task wipes to work
     assert x.tid not in parsl.dfk().tasks, "Task record should be wiped after task completion"

parsl/tests/test_python_apps/test_join.py

@@ -1,17 +1,14 @@
 import pytest
-import time
 
 from parsl import join_app, python_app
 from parsl.dataflow.errors import JoinError
 
-from parsl.tests.configs.local_threads import fresh_config as local_config
 
 RESULT_CONSTANT = 3
 
 
-@python_app(cache=True)
+@python_app
 def inner_app():
-    time.sleep(1)
     return RESULT_CONSTANT
 
 
@@ -34,24 +31,17 @@ def combine(*args):
 
 @join_app
 def outer_make_a_dag_combine(n):
-    futs = []
-    for _ in range(n):
-        futs.append(inner_app())
-    return combine(*futs)
+    return combine(*(inner_app() for _ in range(n)))
 
 
 @join_app
 def outer_make_a_dag_multi(n):
-    futs = []
-    for _ in range(n):
-        futs.append(inner_app())
-    return futs
+    return [inner_app() for _ in range(n)]
 
 
 def test_result_flow():
     f = outer_app()
-    res = f.result()
-    assert res == RESULT_CONSTANT
+    assert f.result() == RESULT_CONSTANT
 
 
 @join_app
@@ -67,20 +57,17 @@ def test_wrong_type():
 
 def test_dependency_on_joined():
     g = add_one(outer_app())
-    res = g.result()
-    assert res == RESULT_CONSTANT + 1
+    assert g.result() == RESULT_CONSTANT + 1
 
 
 def test_combine():
     f = outer_make_a_dag_combine(inner_app())
-    res = f.result()
-    assert res == [RESULT_CONSTANT] * RESULT_CONSTANT
+    assert f.result() == [RESULT_CONSTANT] * RESULT_CONSTANT
 
 
 def test_multiple_return():
     f = outer_make_a_dag_multi(inner_app())
-    res = f.result()
-    assert res == [RESULT_CONSTANT] * RESULT_CONSTANT
+    assert f.result() == [RESULT_CONSTANT] * RESULT_CONSTANT
 
 
 class InnerError(RuntimeError):
@@ -139,3 +126,15 @@ def test_one_error_one_result():
     de0 = e.dependent_exceptions_tids[0][0]
     assert isinstance(de0, InnerError)
     assert de0.args[0] == "Error A"
+
+
+@join_app
+def app_no_futures():
+    return []
+
+
+def test_no_futures():
+    # tests that a list of futures that contains no futures will
+    # complete - regression test for issue #2792
+    f = app_no_futures()
+    assert f.result() == []

parsl/tests/test_python_apps/test_mapred.py

@@ -1,19 +1,15 @@
-import argparse
+import pytest
 
-import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.local_threads import config
 
 
 @python_app
-def fan_out(x, dur):
-    import time
-    time.sleep(dur)
+def times_two(x):
     return x * 2
 
 
 @python_app
-def accumulate(inputs=[]):
+def accumulate(inputs=()):
     return sum(inputs)
 
 
@@ -22,38 +18,17 @@ def accumulate_t(*args):
     return sum(args)
 
 
-def test_mapred_type1(width=2):
-    """MapReduce test with the reduce stage taking futures in inputs=[]
-    """
-
-    futs = []
-    for i in range(1, width + 1):
-        fu = fan_out(i, 1)
-        futs.extend([fu])
-
-    print("Fan out : ", futs)
-
+@pytest.mark.parametrize("width", (2, 3, 5))
+def test_mapred_type1(width):
+    """MapReduce test with the reduce stage taking futures in inputs=[]"""
+    futs = [times_two(i) for i in range(width)]
     red = accumulate(inputs=futs)
-    # print([(i, i.done()) for i in futs])
-    r = sum([x * 2 for x in range(1, width + 1)])
-    assert r == red.result(), "[TEST] MapRed type1 expected %s, got %s" % (
-        r, red.result())
-
+    assert red.result() == 2 * sum(range(width))
 
-def test_mapred_type2(width=2):
-    """MapReduce test with the reduce stage taking futures on the args
-    """
-
-    futs = []
-    for i in range(1, width + 1):
-        fu = fan_out(i, 0.1)
-        futs.extend([fu])
-
-    print("Fan out : ", futs)
 
+@pytest.mark.parametrize("width", (2, 3, 5))
+def test_mapred_type2(width):
+    """MapReduce test with the reduce stage taking futures on the args"""
+    futs = [times_two(i) for i in range(width)]
     red = accumulate_t(*futs)
-
-    # print([(i, i.done()) for i in futs])
-    r = sum([x * 2 for x in range(1, width + 1)])
-    assert r == red.result(), "[TEST] MapRed type2 expected %s, got %s" % (
-        r, red.result())
+    assert red.result() == 2 * sum(range(width))

parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py

@@ -29,9 +29,8 @@ def noop_app(x, inputs=[], cache=True):
 
 
 @python_app
-def sleep(t):
-    import time
-    time.sleep(t)
+def some_func(_t):
+    pass
 
 
 def test_python_unmemoizable():
@@ -51,14 +50,14 @@ def test_python_failing_memoizer():
 
 
 def test_python_unmemoizable_after_dep():
-    sleep_fut = sleep(1)
-    fut = noop_app(Unmemoizable(), inputs=[sleep_fut])
+    memoizable_fut = some_func(1)
+    fut = noop_app(Unmemoizable(), inputs=[memoizable_fut])
     with pytest.raises(ValueError):
         fut.result()
 
 
 def test_python_failing_memoizer_afer_dep():
-    sleep_fut = sleep(1)
-    fut = noop_app(FailingMemoizable(), inputs=[sleep_fut])
+    memoizable_fut = some_func(1)
+    fut = noop_app(FailingMemoizable(), inputs=[memoizable_fut])
     with pytest.raises(ValueError):
         fut.result()

parsl/tests/test_python_apps/test_outputs.py

@@ -1,12 +1,9 @@
-import argparse
 import os
-import pytest
-import shutil
-
 from concurrent.futures import wait
 
+import pytest
+
 from parsl import File, python_app
-from parsl.tests.configs.local_threads import fresh_config as local_config
 
 
 @python_app
@@ -20,22 +17,12 @@ whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs',
 
 
 @pytest.mark.issue363
-def test_launch_apps(n=2, outdir='outputs'):
-    if not os.path.exists(outdir):
-        os.makedirs(outdir)
-    else:
-        shutil.rmtree(outdir)
-        os.makedirs(outdir)
-    print('outdir is ', outdir)
-
-    all_futs = []
-    for i in range(n):
-        fus = double(i, outputs=[File('{0}/{1}.txt'.format(outdir, i))])
-        all_futs.append(fus)
-
-    wait(all_futs)
-
-    stdout_file_count = len(
-        [item for item in os.listdir(outdir) if item.endswith('.txt')])
-    assert stdout_file_count == n, "Only {}/{} files in '{}' ".format(
-            len(os.listdir('outputs/')), n, os.listdir(outdir))
+def test_launch_apps(tmpd_cwd, n=2):
+    outdir = tmpd_cwd / "outputs"
+    outdir.mkdir()
+
+    futs = [double(i, outputs=[File(str(outdir / f"{i}.txt"))]) for i in range(n)]
+    wait(futs)
+
+    stdout_file_count = len(list(outdir.glob("*.txt")))
+    assert stdout_file_count == n, sorted(outdir.glob("*.txt"))

parsl/tests/test_python_apps/test_overview.py

@@ -1,8 +1,4 @@
-import argparse
-
-import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.local_threads import config
 
 
 @python_app
@@ -11,50 +7,17 @@ def app_double(x):
 
 
 @python_app
-def app_sum(inputs=[]):
+def app_sum(inputs=()):
     return sum(inputs)
 
 
-@python_app
-def slow_app_double(x, sleep_dur=0.05):
-    import time
-    time.sleep(sleep_dur)
-    return x * 2
-
-
 def test_1(N=10):
     """Testing code snippet from the documentation
     """
 
-    # Create a list of integers
-    items = range(0, N)
-
-    # Map Phase : Apply an *app* function to each item in list
-    mapped_results = []
-    for i in items:
-        x = app_double(i)
-        mapped_results.append(x)
-
-    total = app_sum(inputs=mapped_results)
-
-    assert total.result() != sum(items), "Sum is wrong {0} != {1}".format(
-        total.result(), sum(items))
-
-
-def test_2(N=10):
-    """Testing code snippet from the documentation
-    """
-
-    # Create a list of integers
-    items = range(0, N)
-
-    # Map Phase : Apply an *app* function to each item in list
-    mapped_results = []
-    for i in items:
-        x = slow_app_double(i)
-        mapped_results.append(x)
+    # Create a list of integers, then apply *app* function to each
+    items = range(N)
+    mapped_results = list(map(app_double, items))
 
     total = app_sum(inputs=mapped_results)
-
-    assert total.result() != sum(items), "Sum is wrong {0} != {1}".format(
-        total.result(), sum(items))
+    assert total.result() == 2 * sum(items)