parsl 2023.7.3__py3-none-any.whl → 2023.7.17__py3-none-any.whl

parsl/tests/test_bash_apps/test_kwarg_storage.py

@@ -1,104 +1,33 @@
-import os
 import pytest
+
 from parsl.app.app import bash_app
 
 
 @bash_app
 def foo(z=2, stdout=None):
-    return """echo {val}
-    """.format(val=z)
+    return f"echo {z}"
 
 
 @pytest.mark.issue363
-def test_command_format_1():
+def test_command_format_1(tmpd_cwd):
     """Testing command format for BashApps
     """
 
-    stdout = os.path.abspath('std.out.0')
-    if os.path.exists(stdout):
-        os.remove(stdout)
-
-    app_fu = foo(stdout=stdout)
-    print("app_fu : ", app_fu)
-    contents = None
-
-    assert app_fu.result() == 0, "BashApp exited with an error code : {0}".format(
-        app_fu.result())
-
-    with open(stdout, 'r') as stdout_f:
-        contents = stdout_f.read()
-        print("Contents : ", contents)
-
-    if os.path.exists('stdout_file'):
-        os.remove(stdout)
-
-    assert contents == '2\n', 'Output does not match expected string "2", Got: "{0}"'.format(
-        contents)
-
-# ===========
-
-    stdout = os.path.abspath('std.out.1')
-    if os.path.exists(stdout):
-        os.remove(stdout)
-
-    app_fu = foo(z=3, stdout=stdout)
-    print("app_fu : ", app_fu)
-    contents = None
-
-    assert app_fu.result() == 0, "BashApp exited with an error code : {0}".format(
-        app_fu.result())
-
-    with open(stdout, 'r') as stdout_f:
-        contents = stdout_f.read()
-        print("Contents : ", contents)
-
-    if os.path.exists('stdout_file'):
-        os.remove(stdout)
-
-    assert contents == '3\n', 'Output does not match expected string "3", Got: "{0}"'.format(
-        contents)
-
-# ===========
-    stdout = os.path.abspath('std.out.2')
-    if os.path.exists(stdout):
-        os.remove(stdout)
-
-    app_fu = foo(z=4, stdout=stdout)
-    print("app_fu : ", app_fu)
-    contents = None
-
-    assert app_fu.result() == 0, "BashApp exited with an error code : {0}".format(
-        app_fu.result())
-
-    with open(stdout, 'r') as stdout_f:
-        contents = stdout_f.read()
-        print("Contents : ", contents)
-
-    if os.path.exists('stdout_file'):
-        os.remove(stdout)
-
-    assert contents == '4\n', 'Output does not match expected string "4", Got: "{0}"'.format(
-        contents)
-
-# ===========
-    stdout = os.path.abspath('std.out.3')
-    if os.path.exists(stdout):
-        os.remove(stdout)
-
-    app_fu = foo(stdout=stdout)
-    print("app_fu : ", app_fu)
-    contents = None
-
-    assert app_fu.result() == 0, "BashApp exited with an error code : {0}".format(
-        app_fu.result())
+    stdout = tmpd_cwd / "std.out"
+    for exp_value, z in (
+        ("3", 3),
+        ("4", 4),
+        ("5", 5),
+    ):
+        app_fu = foo(z=z, stdout=str(stdout))
+        assert app_fu.result() == 0, "BashApp had non-zero exit"
 
-    with open(stdout, 'r') as stdout_f:
-        contents = stdout_f.read()
-        print("Contents : ", contents)
+        so_content = stdout.read_text().strip()
+        assert so_content == exp_value
+        stdout.unlink()
 
-    if os.path.exists('stdout_file'):
-        os.remove(stdout)
+    app_fu = foo(stdout=str(stdout))
+    assert app_fu.result() == 0, "BashApp had non-zero exit"
 
-    assert contents == '2\n', 'Output does not match expected string "2", Got: "{0}"'.format(
-        contents)
-    return True
+    so_content = stdout.read_text().strip()
+    assert so_content == "2"

parsl/tests/test_bash_apps/test_memoize.py

@@ -1,15 +1,11 @@
-import argparse
-import os
 import pytest
 
-import parsl
 from parsl import File
 from parsl.app.app import bash_app
-from parsl.tests.configs.local_threads import config
 
 
 @bash_app(cache=True)
-def fail_on_presence(outputs=[]):
+def fail_on_presence(outputs=()):
     return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])
 
 
@@ -17,31 +13,20 @@ def fail_on_presence(outputs=[]):
 # won't work if there's a staging provider.
 # @pytest.mark.sharedFS_required
 @pytest.mark.issue363
-def test_bash_memoization(n=2):
+def test_bash_memoization(tmpd_cwd, n=2):
     """Testing bash memoization
     """
-    temp_filename = "test.memoization.tmp"
-    temp_file = File(temp_filename)
+    mpath = tmpd_cwd / "test.memoization.tmp"
+    temp_file = File(str(mpath))
+    fail_on_presence(outputs=[temp_file]).result()
 
-    if os.path.exists(temp_filename):
-        os.remove(temp_filename)
-
-    temp_file = File(temp_filename)
-
-    print("Launching: ", n)
-    x = fail_on_presence(outputs=[temp_file])
-    x.result()
-
-    d = {}
-    for i in range(0, n):
-        d[i] = fail_on_presence(outputs=[temp_file])
-
-    for i in d:
-        assert d[i].exception() is None
+    futs = [fail_on_presence(outputs=[temp_file]) for _ in range(n)]
+    for f in futs:
+        assert f.exception() is None
 
 
 @bash_app(cache=True)
-def fail_on_presence_kw(outputs=[], foo={}):
+def fail_on_presence_kw(outputs=(), foo=None):
     return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])
 
 
@@ -49,24 +34,15 @@ def fail_on_presence_kw(outputs=[], foo={}):
 # won't work if there's a staging provider.
 # @pytest.mark.sharedFS_required
 @pytest.mark.issue363
-def test_bash_memoization_keywords(n=2):
+def test_bash_memoization_keywords(tmpd_cwd, n=2):
     """Testing bash memoization
     """
-    temp_filename = "test.memoization.tmp"
-    temp_file = File("test.memoization.tmp")
-
-    if os.path.exists(temp_filename):
-        os.remove(temp_filename)
-
-    temp_file = File(temp_filename)
-
-    print("Launching: ", n)
-    x = fail_on_presence_kw(outputs=[temp_file], foo={"a": 1, "b": 2})
-    x.result()
+    mpath = tmpd_cwd / "test.memoization.tmp"
+    temp_file = File(str(mpath))
 
-    d = {}
-    for i in range(0, n):
-        d[i] = fail_on_presence_kw(outputs=[temp_file], foo={"b": 2, "a": 1})
+    foo = {"a": 1, "b": 2}
+    fail_on_presence_kw(outputs=[temp_file], foo=foo).result()
 
-    for i in d:
-        assert d[i].exception() is None
+    futs = [fail_on_presence_kw(outputs=[temp_file], foo=foo) for _ in range(n)]
+    for f in futs:
+        assert f.exception() is None

parsl/tests/test_bash_apps/test_multiline.py

@@ -1,21 +1,11 @@
-import argparse
-import os
 import pytest
-import shutil
-import time
 
-import parsl
 from parsl import File
 from parsl.app.app import bash_app
-from parsl.tests.configs.local_threads import config
 
 
 @bash_app
-def multiline(
-        inputs=[],
-        outputs=[],
-        stderr=os.path.abspath('std.err'),
-        stdout=os.path.abspath('std.out')):
+def multiline(inputs=(), outputs=(), stderr=None, stdout=None):
     return """echo {inputs[0]} &> {outputs[0]}
     echo {inputs[1]} &> {outputs[1]}
     echo {inputs[2]} &> {outputs[2]}
@@ -25,39 +15,23 @@ def multiline(
 
 
 @pytest.mark.issue363
-def test_multiline():
-
-    outdir = os.path.abspath('outputs')
-
-    if not os.path.exists(outdir):
-        os.makedirs(outdir)
-    else:
-        shutil.rmtree(outdir)
-        os.makedirs(outdir)
-
+def test_multiline(tmpd_cwd):
+    so, se = tmpd_cwd / "std.out", tmpd_cwd / "std.err"
     f = multiline(
-            inputs=["Hello", "This is", "Cat!"],
-            outputs=[
-                File('{0}/hello.txt'.format(outdir)),
-                File('{0}/this.txt'.format(outdir)),
-                File('{0}/cat.txt'.format(outdir))
-            ]
+        inputs=["Hello", "This is", "Cat!"],
+        outputs=[
+            File(str(tmpd_cwd / "hello.txt")),
+            File(str(tmpd_cwd / "this.txt")),
+            File(str(tmpd_cwd / "cat.txt")),
+        ],
+        stdout=str(so),
+        stderr=str(se),
     )
-    print(f.result())
-
-    time.sleep(0.1)
-    assert 'hello.txt' in os.listdir(outdir), "hello.txt is missing"
-    assert 'this.txt' in os.listdir(outdir), "this.txt is missing"
-    assert 'cat.txt' in os.listdir(outdir), "cat.txt is missing"
-
-    with open('std.out', 'r') as o:
-        out = o.read()
-        assert out != "Testing STDOUT", "Stdout is bad"
-
-    with open('std.err', 'r') as o:
-        err = o.read()
-        assert err != "Testing STDERR", "Stderr is bad"
-
-    os.remove('std.err')
-    os.remove('std.out')
-    return True
+    f.result()
+
+    flist = list(map(str, (f.name for f in tmpd_cwd.iterdir())))
+    assert 'hello.txt' in flist, "hello.txt is missing"
+    assert 'this.txt' in flist, "this.txt is missing"
+    assert 'cat.txt' in flist, "cat.txt is missing"
+    assert "Testing STDOUT" in so.read_text()
+    assert "Testing STDERR" in se.read_text()

parsl/tests/test_bash_apps/test_pipeline.py

@@ -1,17 +1,12 @@
-import argparse
-import os
 import pytest
 
-import parsl
 from parsl.app.app import bash_app
 from parsl.data_provider.files import File
 from parsl.app.futures import DataFuture
 
-from parsl.tests.configs.local_threads import config
-
 
 @bash_app
-def increment(inputs=[], outputs=[], stdout=None, stderr=None):
+def increment(inputs=(), outputs=(), stdout=None, stderr=None):
     cmd_line = """
     if ! [ -f {inputs[0]} ] ; then exit 43 ; fi
     x=$(cat {inputs[0]})
@@ -21,7 +16,7 @@ def increment(inputs=[], outputs=[], stdout=None, stderr=None):
 
 
 @bash_app
-def slow_increment(dur, inputs=[], outputs=[], stdout=None, stderr=None):
+def slow_increment(dur, inputs=(), outputs=(), stdout=None, stderr=None):
     cmd_line = """
     x=$(cat {inputs[0]})
     echo $(($x+1)) > {outputs[0]}
@@ -30,91 +25,60 @@ def slow_increment(dur, inputs=[], outputs=[], stdout=None, stderr=None):
     return cmd_line
 
 
-def cleanup_work(depth):
-    for i in range(0, depth):
-        fn = "test{0}.txt".format(i)
-        if os.path.exists(fn):
-            os.remove(fn)
-
-
 @pytest.mark.staging_required
-def test_increment(depth=5):
+def test_increment(tmpd_cwd, depth=5):
     """Test simple pipeline A->B...->N
     """
+    fpath = tmpd_cwd / "test0.txt"
+    fpath.write_text("0\n")
 
-    cleanup_work(depth)
-
-    # Create the first file
-    open("test0.txt", 'w').write('0\n')
-
-    # Create the first entry in the dictionary holding the futures
-    prev = File("test0.txt")
-    futs = {}
+    prev = [File(str(fpath))]
+    futs = []
     for i in range(1, depth):
-        print("Launching {0} with {1}".format(i, prev))
-        assert isinstance(prev, DataFuture) or isinstance(prev, File)
-        output = File("test{0}.txt".format(i))
-        fu = increment(inputs=[prev],  # Depend on the future from previous call
-                       # Name the file to be created here
-                       outputs=[output],
-                       stdout="incr{0}.out".format(i),
-                       stderr="incr{0}.err".format(i))
-        [prev] = fu.outputs
-        futs[i] = prev
-        print(prev.filepath)
-        assert isinstance(prev, DataFuture)
-
-    for key in futs:
-        if key > 0:
-            fu = futs[key]
-            file = fu.result()
-            filename = file.filepath
-
-            # this test is a bit close to a test of the specific implementation
-            # of File
-            assert file.local_path is None, "File on local side has overridden local_path, file: {}".format(repr(file))
-            assert file.filepath == "test{0}.txt".format(key), "Submit side filepath has not been preserved over execution"
-
-            data = open(filename, 'r').read().strip()
-            assert data == str(
-                key), "[TEST] incr failed for key: {0} got data: {1} from filename {2}".format(key, data, filename)
-
-    cleanup_work(depth)
+        assert isinstance(prev[0], (DataFuture, File))
+        output = File(str(tmpd_cwd / f"test{i}.txt"))
+        f = increment(
+            inputs=prev,
+            outputs=[output],
+            stdout=str(tmpd_cwd / f"incr{i}.out"),
+            stderr=str(tmpd_cwd / f"incr{i}.err"),
+        )
+        prev = f.outputs
+        futs.append((i, prev[0]))
+        assert isinstance(prev[0], DataFuture)
+
+    for key, f in futs:
+        file = f.result()
+        expected = str(tmpd_cwd / f"test{key}.txt")
+
+        assert file.local_path is None, "File on local side has overridden local_path, file: {}".format(repr(file))
+        assert file.filepath == expected, "Submit side filepath has not been preserved over execution"
+        data = open(file.filepath).read().strip()
+        assert data == str(key)
 
 
 @pytest.mark.staging_required
-def test_increment_slow(depth=5, dur=0.5):
+def test_increment_slow(tmpd_cwd, depth=5, dur=0.01):
     """Test simple pipeline slow (sleep.5) A->B...->N
     """
 
-    cleanup_work(depth)
+    fpath = tmpd_cwd / "test0.txt"
+    fpath.write_text("0\n")
 
-    # Create the first file
-    open("test0.txt", 'w').write('0\n')
-
-    prev = File("test0.txt")
-    # Create the first entry in the dictionary holding the futures
-    futs = {}
-    print("************** Type: ", type(dur), dur)
+    prev = [File(str(fpath))]
+    futs = []
     for i in range(1, depth):
-        print("Launching {0} with {1}".format(i, prev))
-        output = File("test{0}.txt".format(i))
-        fu = slow_increment(dur,
-                            # Depend on the future from previous call
-                            inputs=[prev],
-                            # Name the file to be created here
-                            outputs=[output],
-                            stdout="incr{0}.out".format(i),
-                            stderr="incr{0}.err".format(i))
-        [prev] = fu.outputs
-        futs[i] = prev
-        print(prev.filepath)
-
-    for key in futs:
-        if key > 0:
-            fu = futs[key]
-            data = open(fu.result().filepath, 'r').read().strip()
-            assert data == str(
-                key), "[TEST] incr failed for key: {0} got: {1}".format(key, data)
-
-    cleanup_work(depth)
+        output = File(str(tmpd_cwd / f"test{i}.txt"))
+        f = slow_increment(
+            dur,
+            inputs=prev,
+            outputs=[output],
+            stdout=str(tmpd_cwd / f"incr{i}.out"),
+            stderr=str(tmpd_cwd / f"incr{i}.err"),
+        )
+        prev = f.outputs
+        futs.append((i, prev[0]))
+
+    for key, f in futs:
+        data = open(f.result().filepath).read().strip()
+        assert data == str(key)

parsl/tests/test_bash_apps/test_stdout.py

@@ -1,16 +1,13 @@
-import argparse
 import os
 
 import pytest
 
-import parsl
 import parsl.app.errors as perror
 from parsl.app.app import bash_app
-from parsl.tests.configs.local_threads import config
 
 
 @bash_app
-def echo_to_streams(msg, stderr='std.err', stdout='std.out'):
+def echo_to_streams(msg, stderr=None, stdout=None):
     return 'echo "{0}"; echo "{0}" >&2'.format(msg)
 
 
@@ -40,8 +37,7 @@ testids = [
 @pytest.mark.issue363
 @pytest.mark.parametrize('spec', speclist, ids=testids)
 def test_bad_stdout_specs(spec):
-    """Testing bad stdout spec cases
-    """
+    """Testing bad stdout spec cases"""
 
     fn = echo_to_streams("Hello world", stdout=spec, stderr='t.err')
 
@@ -52,18 +48,14 @@ def test_bad_stdout_specs(spec):
     else:
         assert False, "Did not raise expected exception"
 
-    return
-
 
 @pytest.mark.issue363
 def test_bad_stderr_file():
+    """Testing bad stderr file"""
 
-    """ Testing bad stderr file """
-
-    out = "t2.out"
     err = "/bad/dir/t2.err"
 
-    fn = echo_to_streams("Hello world", stdout=out, stderr=err)
+    fn = echo_to_streams("Hello world", stderr=err)
 
     try:
         fn.result()
@@ -76,13 +68,11 @@ def test_bad_stderr_file():
 
 
 @pytest.mark.issue363
-def test_stdout_truncate():
-
-    """ Testing truncation of prior content of stdout """
+def test_stdout_truncate(tmpd_cwd):
+    """Testing truncation of prior content of stdout"""
 
-    out = ('t1.out', 'w')
-    err = 't1.err'
-    os.system('rm -f ' + out[0] + ' ' + err)
+    out = (str(tmpd_cwd / 't1.out'), 'w')
+    err = str(tmpd_cwd / 't1.err')
 
     echo_to_streams('hi', stdout=out, stderr=err).result()
     len1 = len(open(out[0]).readlines())
@@ -90,19 +80,16 @@ def test_stdout_truncate():
     echo_to_streams('hi', stdout=out, stderr=err).result()
     len2 = len(open(out[0]).readlines())
 
-    assert len1 == len2 == 1, "Line count of output files should both be 1, but: len1={} len2={}".format(len1, len2)
-
-    os.system('rm -f ' + out[0] + ' ' + err)
+    assert len1 == 1
+    assert len1 == len2
 
 
 @pytest.mark.issue363
-def test_stdout_append():
-
-    """ Testing appending to prior content of stdout (default open() mode) """
+def test_stdout_append(tmpd_cwd):
+    """Testing appending to prior content of stdout (default open() mode)"""
 
-    out = 't1.out'
-    err = 't1.err'
-    os.system('rm -f ' + out + ' ' + err)
+    out = str(tmpd_cwd / 't1.out')
+    err = str(tmpd_cwd / 't1.err')
 
     echo_to_streams('hi', stdout=out, stderr=err).result()
     len1 = len(open(out).readlines())
@@ -110,6 +97,4 @@ def test_stdout_append():
     echo_to_streams('hi', stdout=out, stderr=err).result()
     len2 = len(open(out).readlines())
 
-    assert len1 == 1 and len2 == 2, "Line count of output files should be 1 and 2, but:  len1={} len2={}".format(len1, len2)
-
-    os.system('rm -f ' + out + ' ' + err)
+    assert len1 == 1 and len2 == 2

parsl/tests/test_data/test_file_apps.py

@@ -12,13 +12,13 @@ def cat(inputs=(), outputs=(), stdout=None, stderr=None):
 
 @pytest.mark.staging_required
 def test_files(setup_data):
-    fs = sorted(str(setup_data / f) for f in setup_data.iterdir())
+    fs = sorted(setup_data / f for f in setup_data.iterdir())
     fs = list(map(File, fs))
     x = cat(
         inputs=fs,
-        outputs=[File(str(setup_data / "cat_out.txt"))],
-        stdout=str(setup_data / "f_app.out"),
-        stderr=str(setup_data / "f_app.err"),
+        outputs=[File(setup_data / "cat_out.txt")],
+        stdout=setup_data / "f_app.out",
+        stderr=setup_data / "f_app.err",
     )
     x.result()
     d_x = x.outputs[0]
@@ -28,29 +28,27 @@ def test_files(setup_data):
 
 @bash_app
 def increment(inputs=(), outputs=(), stdout=None, stderr=None):
-    # Place double braces to avoid python complaining about missing keys for {item = $1}
-    return """
-    x=$(cat {i})
-    echo $(($x+1)) > {o}
-    """.format(i=inputs[0], o=outputs[0])
+    return (
+        f"x=$(cat {inputs[0]})\n"
+        f"echo $(($x+1)) > {outputs[0]}"
+    )
 
 
 @pytest.mark.staging_required
 def test_increment(tmp_path, depth=5):
-    """Test simple pipeline A->B...->N
-    """
+    """Test simple pipeline A->B...->N"""
     # Test setup
     first_fpath = tmp_path / "test0.txt"
     first_fpath.write_text("0\n")
 
-    prev = [File(str(first_fpath))]
+    prev = [File(first_fpath)]
     futs = []
     for i in range(1, depth):
         f = increment(
             inputs=prev,
-            outputs=[File(str(tmp_path / f"test{i}.txt"))],
-            stdout=str(tmp_path / f"incr{i}.out"),
-            stderr=str(tmp_path / f"incr{i}.err"),
+            outputs=[File(tmp_path / f"test{i}.txt")],
+            stdout=tmp_path / f"incr{i}.out",
+            stderr=tmp_path / f"incr{i}.err",
         )
         prev = f.outputs
         futs.append((i, prev[0]))

parsl/tests/test_data/test_file_staging.py

@@ -7,7 +7,7 @@ from parsl.data_provider.files import File
 @bash_app
 def cat(inputs=(), outputs=(), stdout=None, stderr=None):
     infiles = " ".join(i.filepath for i in inputs)
-    return f"cat {infiles} &> {outputs[0]}\n"
+    return f"cat {infiles} &> {outputs[0]}"
 
 
 @pytest.mark.staging_required
@@ -17,7 +17,7 @@ def test_regression_200(tmp_path):
     fpath = tmp_path / "test.txt"
 
     fpath.write_text("Hello World")
-    f = cat(inputs=[File(str(fpath))], outputs=[File(str(opath))])
+    f = cat(inputs=[File(fpath)], outputs=[File(opath)])
 
     f.result()
     with open(f.outputs[0].filepath) as f: