parsl 2024.3.11__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/tests/scaling_tests/test_scale.py

@@ -4,15 +4,6 @@ import argparse
 import time
 
 import parsl
-
-# from parsl.tests.configs.htex_local import config
-# from htex_local import config
-# from parsl.configs.local_threads import config
-# from parsl.configs.local_ipp import config
-
-# parsl.set_stream_logger()
-# config.executors[0].provider.tasks_per_node = 4
-# parsl.load(config)
 from parsl.app.app import python_app  # , bash_app
 
 

parsl/tests/scaling_tests/vineex_condor.py

@@ -1,6 +1,5 @@
 from parsl.config import Config
-from parsl.executors.taskvine import TaskVineExecutor
-from parsl.executors.taskvine import TaskVineManagerConfig
+from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
 from parsl.providers import CondorProvider
 
 config = Config(

parsl/tests/scaling_tests/vineex_local.py

@@ -1,6 +1,5 @@
 from parsl.config import Config
-from parsl.executors.taskvine import TaskVineExecutor
-from parsl.executors.taskvine import TaskVineManagerConfig
+from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
 from parsl.providers import LocalProvider
 
 config = Config(

parsl/tests/site_tests/site_config_selector.py

@@ -7,12 +7,7 @@ def fresh_config():
     hostname = os.getenv('PARSL_HOSTNAME', platform.uname().node)
     print("Loading config for {}".format(hostname))
 
-    if 'thetalogin' in hostname:
-        from parsl.tests.configs.theta import fresh_config
-        config = fresh_config()
-        print("Loading Theta config")
-
-    elif 'frontera' in hostname:
+    if 'frontera' in hostname:
         print("Loading Frontera config")
         from parsl.tests.configs.frontera import fresh_config
         config = fresh_config()

parsl/tests/site_tests/test_provider.py

@@ -1,8 +1,10 @@
 import argparse
 import logging
+import time
+
 import pytest
+
 import parsl
-import time
 from parsl.app.app import python_app  # , bash_app
 from parsl.jobs.states import JobState
 from parsl.tests.site_tests.site_config_selector import fresh_config
@@ -58,7 +60,7 @@ def test_provider():
     logger.info("Job in terminal state")
 
     _, current_jobs = executor._get_block_and_job_ids()
-    # PR 1952 stoped removing scale_in blocks from self.blocks
+    # PR 1952 stoped removing scale_in blocks from self.blocks_to_job_id
     # A new PR will handle removing blocks from self.block
     # this includes failed/completed/canceled blocks
     assert len(current_jobs) == 1, "Expected current_jobs == 1"

parsl/tests/site_tests/test_site.py

@@ -1,5 +1,7 @@
 import argparse
+
 import pytest
+
 import parsl
 from parsl.app.app import python_app  # , bash_app
 from parsl.tests.site_tests.site_config_selector import fresh_config

parsl/tests/sites/test_affinity.py

@@ -1,12 +1,13 @@
 """Tests related to assigning workers to specific compute units"""
 
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
+import os
+
+import pytest
+
+from parsl import python_app
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-from parsl import python_app
-import pytest
-import os
+from parsl.providers import LocalProvider
 
 
 def local_config():
@@ -20,7 +21,6 @@ def local_config():
                 available_accelerators=2,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     max_blocks=1,
                 ),
@@ -32,8 +32,8 @@ def local_config():
 
 @python_app
 def get_worker_info():
-    from time import sleep
     import os
+    from time import sleep
     rank = int(os.environ['PARSL_WORKER_RANK'])
     aff = os.sched_getaffinity(0)
     device = os.environ.get('CUDA_VISIBLE_DEVICES')

parsl/tests/sites/test_dynamic_executor.py

@@ -1,10 +1,10 @@
+import pytest
+
 import parsl
+from parsl.app.app import python_app
 from parsl.executors import HighThroughputExecutor
 from parsl.executors.threads import ThreadPoolExecutor
 from parsl.providers import LocalProvider
-from parsl.app.app import python_app
-
-import pytest
 
 
 @python_app(executors=['threads'])
@@ -75,4 +75,3 @@ def test_dynamic_executor():
     print("Done testing")
 
     dfk.cleanup()
-    parsl.clear()

parsl/tests/sites/test_ec2.py

@@ -1,10 +1,11 @@
-import parsl
+import logging
+
 import pytest
 
+import parsl
 from parsl.app.app import python_app
 from parsl.tests.configs.ec2_single_node import config
 
-import logging
 logger = logging.getLogger(__name__)
 
 local_config = config

parsl/tests/sites/test_worker_info.py

@@ -1,11 +1,11 @@
 """Tests related to Parsl workers being able to access their worker ID"""
 
-from parsl.providers import LocalProvider
-from parsl.channels import LocalChannel
+import pytest
+
+from parsl import python_app
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-from parsl import python_app
-import pytest
+from parsl.providers import LocalProvider
 
 
 def local_config():
@@ -17,7 +17,6 @@ def local_config():
                 max_workers_per_node=4,
                 encrypted=True,
                 provider=LocalProvider(
-                    channel=LocalChannel(),
                     init_blocks=1,
                     max_blocks=1,
                 ),

parsl/tests/test_aalst_patterns.py

@@ -6,7 +6,6 @@ import pytest
 from parsl.app.app import python_app
 from parsl.tests.configs.local_threads import config
 
-
 pytestmark = pytest.mark.skip('not asserting anything')
 
 

parsl/tests/test_bash_apps/test_apptimeout.py

@@ -1,9 +1,9 @@
-import parsl
 import pytest
 
+import parsl
 from parsl.app.app import bash_app
-from parsl.tests.configs.local_threads import config
 from parsl.app.errors import AppTimeout
+from parsl.tests.configs.local_threads import config
 
 
 @bash_app

parsl/tests/test_bash_apps/test_basic.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import random
 import re
@@ -23,7 +24,7 @@ def foo(x, y, z=10, stdout=None, label=None):
     return f"echo {x} {y} {z}"
 
 
-@pytest.mark.issue363
+@pytest.mark.shared_fs
 def test_command_format_1(tmpd_cwd):
     """Testing command format for BashApps"""
 
@@ -38,8 +39,8 @@ def test_command_format_1(tmpd_cwd):
     assert so_content == "1 4 10"
 
 
-@pytest.mark.issue363
-def test_auto_log_filename_format():
+@pytest.mark.shared_fs
+def test_auto_log_filename_format(caplog):
     """Testing auto log filename format for BashApps
     """
     app_label = "label_test_auto_log_filename_format"
@@ -51,6 +52,8 @@ def test_auto_log_filename_format():
         foo_future.result())
 
     log_fpath = foo_future.stdout
+    assert isinstance(log_fpath, str)
+
     log_pattern = fr".*/task_\d+_foo_{app_label}"
     assert re.match(log_pattern, log_fpath), 'Output file "{0}" does not match pattern "{1}"'.format(
         log_fpath, log_pattern)
@@ -61,8 +64,11 @@ def test_auto_log_filename_format():
     assert contents == '1 {0} 10\n'.format(rand_int), \
         'Output does not match expected string "1 {0} 10", Got: "{1}"'.format(rand_int, contents)
 
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR
+
 
-@pytest.mark.issue363
+@pytest.mark.shared_fs
 def test_parallel_for(tmpd_cwd, n=3):
     """Testing a simple parallel for loop"""
     outdir = tmpd_cwd / "outputs/test_parallel"

parsl/tests/test_bash_apps/test_error_codes.py

@@ -4,12 +4,9 @@ import os
 import pytest
 
 import parsl
-from parsl.app.app import bash_app
 import parsl.app.errors as pe
-
-
+from parsl.app.app import bash_app
 from parsl.app.errors import BashExitFailure
-
 from parsl.tests.configs.local_threads import fresh_config as local_config
 
 
@@ -61,6 +58,7 @@ test_matrix = {
 whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs', '*threads*')
 
 
+@pytest.mark.shared_fs
 def test_div_0(test_fn=div_0):
     err_code = test_matrix[test_fn]['exit_code']
     f = test_fn()
@@ -76,7 +74,7 @@ def test_div_0(test_fn=div_0):
     os.remove('std.out')
 
 
-@pytest.mark.issue363
+@pytest.mark.shared_fs
 def test_bash_misuse(test_fn=bash_misuse):
     err_code = test_matrix[test_fn]['exit_code']
     f = test_fn()
@@ -91,7 +89,7 @@ def test_bash_misuse(test_fn=bash_misuse):
     os.remove('std.out')
 
 
-@pytest.mark.issue363
+@pytest.mark.shared_fs
 def test_command_not_found(test_fn=command_not_found):
     err_code = test_matrix[test_fn]['exit_code']
     f = test_fn()
@@ -108,7 +106,7 @@ def test_command_not_found(test_fn=command_not_found):
     return True
 
 
-@pytest.mark.issue363
+@pytest.mark.shared_fs
 def test_not_executable(test_fn=not_executable):
     err_code = test_matrix[test_fn]['exit_code']
     f = test_fn()

parsl/tests/test_bash_apps/test_inputs_default.py

@@ -0,0 +1,25 @@
+import pytest
+
+from parsl import AUTO_LOGNAME, Config, bash_app, python_app
+from parsl.executors import ThreadPoolExecutor
+
+
+def local_config():
+    return Config(executors=[ThreadPoolExecutor()])
+
+
+@pytest.mark.local
+def test_default_inputs():
+    @python_app
+    def identity(inp):
+        return inp
+
+    @bash_app
+    def sum_inputs(inputs=[identity(1), identity(2)], stdout=AUTO_LOGNAME):
+        calc = sum(inputs)
+        return f"echo {calc}"
+
+    fut = sum_inputs()
+    fut.result()
+    with open(fut.stdout, 'r') as f:
+        assert int(f.read()) == 3

parsl/tests/test_bash_apps/test_kwarg_storage.py

@@ -8,7 +8,7 @@ def foo(z=2, stdout=None):
     return f"echo {z}"
 
 
-@pytest.mark.issue363
+@pytest.mark.shared_fs
 def test_command_format_1(tmpd_cwd):
     """Testing command format for BashApps
     """

parsl/tests/test_bash_apps/test_memoize.py

@@ -9,10 +9,7 @@ def fail_on_presence(outputs=()):
     return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])
 
 
-# This test is an oddity that requires a shared-FS and simply
-# won't work if there's a staging provider.
-# @pytest.mark.sharedFS_required
-@pytest.mark.issue363
+@pytest.mark.shared_fs
 def test_bash_memoization(tmpd_cwd, n=2):
     """Testing bash memoization
     """
@@ -30,10 +27,7 @@ def fail_on_presence_kw(outputs=(), foo=None):
     return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])
 
 
-# This test is an oddity that requires a shared-FS and simply
-# won't work if there's a staging provider.
-# @pytest.mark.sharedFS_required
-@pytest.mark.issue363
+@pytest.mark.shared_fs
 def test_bash_memoization_keywords(tmpd_cwd, n=2):
     """Testing bash memoization
     """

parsl/tests/test_bash_apps/test_memoize_ignore_args.py

@@ -1,4 +1,5 @@
 import os
+
 import pytest
 
 import parsl
@@ -22,25 +23,19 @@ def no_checkpoint_stdout_app_ignore_args(stdout=None):
     return "echo X"
 
 
-@pytest.mark.issue363
-def test_memo_stdout():
+@pytest.mark.shared_fs
+def test_memo_stdout(tmpd_cwd):
+    path_x = tmpd_cwd / "test.memo.stdout.x"
 
     # this should run and create a file named after path_x
-    path_x = "test.memo.stdout.x"
-    if os.path.exists(path_x):
-        os.remove(path_x)
-
-    no_checkpoint_stdout_app_ignore_args(stdout=path_x).result()
-    assert os.path.exists(path_x)
-
-    # this should be memoized, so not create benc.test.y
-    path_y = "test.memo.stdout.y"
+    no_checkpoint_stdout_app_ignore_args(stdout=str(path_x)).result()
+    assert path_x.exists()
 
-    if os.path.exists(path_y):
-        os.remove(path_y)
+    # this should be memoized, so should not get created
+    path_y = tmpd_cwd / "test.memo.stdout.y"
 
     no_checkpoint_stdout_app_ignore_args(stdout=path_y).result()
-    assert not os.path.exists(path_y)
+    assert not path_y.exists(), "For memoization, expected NO file written"
 
     # this should also be memoized, so not create an arbitrary name
     z_fut = no_checkpoint_stdout_app_ignore_args(stdout=parsl.AUTO_LOGNAME)

parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py

@@ -1,9 +1,8 @@
 import copy
-import os
-import pytest
-
 from typing import List
 
+import pytest
+
 import parsl
 from parsl.app.app import bash_app
 
@@ -30,22 +29,18 @@ def no_checkpoint_stdout_app(stdout=None):
     return "echo X"
 
 
-@pytest.mark.issue363
-def test_memo_stdout():
-
+@pytest.mark.shared_fs
+def test_memo_stdout(tmpd_cwd):
     assert const_list_x == const_list_x_arg
 
-    path_x = "test.memo.stdout.x"
-    if os.path.exists(path_x):
-        os.remove(path_x)
+    path_x = tmpd_cwd / "test.memo.stdout.x"
 
     # this should run and create a file named after path_x
-    no_checkpoint_stdout_app(stdout=path_x).result()
-    assert os.path.exists(path_x)
+    no_checkpoint_stdout_app(stdout=str(path_x)).result()
+    path_x.unlink(missing_ok=False)
 
-    os.remove(path_x)
-    no_checkpoint_stdout_app(stdout=path_x).result()
-    assert not os.path.exists(path_x)
+    no_checkpoint_stdout_app(stdout=str(path_x)).result()
+    assert not path_x.exists(), "For memoization, expected NO file written"
 
     # this should also be memoized, so not create an arbitrary name
     z_fut = no_checkpoint_stdout_app(stdout=parsl.AUTO_LOGNAME)

parsl/tests/test_bash_apps/test_multiline.py

@@ -14,7 +14,7 @@ def multiline(inputs=(), outputs=(), stderr=None, stdout=None):
     """.format(inputs=inputs, outputs=outputs)
 
 
-@pytest.mark.issue363
+@pytest.mark.shared_fs
 def test_multiline(tmpd_cwd):
     so, se = tmpd_cwd / "std.out", tmpd_cwd / "std.err"
     f = multiline(

parsl/tests/test_bash_apps/test_pipeline.py

@@ -1,8 +1,8 @@
 import pytest
 
 from parsl.app.app import bash_app
-from parsl.data_provider.files import File
 from parsl.app.futures import DataFuture
+from parsl.data_provider.files import File
 
 
 @bash_app

parsl/tests/test_bash_apps/test_std_uri.py

@@ -0,0 +1,123 @@
+import logging
+import zipfile
+from functools import partial
+
+import pytest
+
+import parsl
+from parsl.app.futures import DataFuture
+from parsl.data_provider.files import File
+from parsl.executors import ThreadPoolExecutor
+
+
+@parsl.bash_app
+def app_stdout(stdout=parsl.AUTO_LOGNAME):
+    return "echo hello"
+
+
+def const_str(cpath, task_record, err_or_out):
+    return cpath
+
+
+def const_with_cpath(autopath_specifier, content_path, caplog):
+    with parsl.load(parsl.Config(std_autopath=partial(const_str, autopath_specifier))):
+        fut = app_stdout()
+
+        # we don't have to wait for a result to check this attributes
+        assert fut.stdout is autopath_specifier
+
+        # there is no DataFuture to wait for in the str case: the model is that
+        # the stdout will be immediately available on task completion.
+        fut.result()
+
+    with open(content_path, "r") as file:
+        assert file.readlines() == ["hello\n"]
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR
+
+
+@pytest.mark.local
+def test_std_autopath_const_str(caplog, tmpd_cwd):
+    """Tests str and tuple mode autopaths with constant autopath, which should
+    all be passed through unmodified.
+    """
+    cpath = str(tmpd_cwd / "CONST")
+    const_with_cpath(cpath, cpath, caplog)
+
+
+@pytest.mark.local
+def test_std_autopath_const_pathlike(caplog, tmpd_cwd):
+    cpath = tmpd_cwd / "CONST"
+    const_with_cpath(cpath, cpath, caplog)
+
+
+@pytest.mark.local
+def test_std_autopath_const_tuples(caplog, tmpd_cwd):
+    file = tmpd_cwd / "CONST"
+    cpath = (file, "w")
+    const_with_cpath(cpath, file, caplog)
+
+
+class URIFailError(Exception):
+    pass
+
+
+def fail_uri(task_record, err_or_out):
+    raise URIFailError("Deliberate failure in std stream filename generation")
+
+
+@pytest.mark.local
+def test_std_autopath_fail(caplog):
+    with parsl.load(parsl.Config(std_autopath=fail_uri)):
+        with pytest.raises(URIFailError):
+            app_stdout()
+
+
+@parsl.bash_app
+def app_both(stdout=parsl.AUTO_LOGNAME, stderr=parsl.AUTO_LOGNAME):
+    return "echo hello; echo goodbye >&2"
+
+
+def zip_uri(base, task_record, err_or_out):
+    """Should generate Files in base.zip like app_both.0.out or app_both.123.err"""
+    zip_path = base / "base.zip"
+    file = f"{task_record['func_name']}.{task_record['id']}.{task_record['try_id']}.{err_or_out}"
+    return File(f"zip:{zip_path}/{file}")
+
+
+@pytest.mark.local
+def test_std_autopath_zip(caplog, tmpd_cwd):
+    with parsl.load(parsl.Config(run_dir=str(tmpd_cwd),
+                                 executors=[ThreadPoolExecutor(working_dir=str(tmpd_cwd))],
+                                 std_autopath=partial(zip_uri, tmpd_cwd))):
+        futs = []
+
+        for _ in range(10):
+            fut = app_both()
+
+            # assertions that should hold after submission
+            assert isinstance(fut.stdout, DataFuture)
+            assert fut.stdout.file_obj.url.startswith("zip")
+
+            futs.append(fut)
+
+        # Barrier for all the stageouts to complete so that we can
+        # poke at the zip file.
+        [(fut.stdout.result(), fut.stderr.result()) for fut in futs]
+
+        with zipfile.ZipFile(tmpd_cwd / "base.zip") as z:
+            for fut in futs:
+
+                assert fut.done(), "AppFuture should be done if stageout is done"
+
+                stdout_relative_path = f"app_both.{fut.tid}.0.stdout"
+                with z.open(stdout_relative_path) as f:
+                    assert f.readlines() == [b'hello\n']
+
+                stderr_relative_path = f"app_both.{fut.tid}.0.stderr"
+                with z.open(stderr_relative_path) as f:
+                    assert f.readlines()[-1] == b'goodbye\n'
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR

parsl/tests/test_bash_apps/test_stdout.py

@@ -1,3 +1,4 @@
+import logging
 import os
 
 import pytest
@@ -15,7 +16,6 @@ def echo_to_streams(msg, stderr=None, stdout=None):
 whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs', '*threads*')
 
 speclist = (
-    '/bad/dir/t.out',
     ['t3.out', 'w'],
     ('t4.out', None),
     (42, 'w'),
@@ -25,7 +25,6 @@ speclist = (
 )
 
 testids = [
-    'nonexistent_dir',
     'list_not_tuple',
     'null_mode',
     'not_a_string',
@@ -35,7 +34,6 @@ testids = [
 ]
 
 
-@pytest.mark.issue363
 @pytest.mark.parametrize('spec', speclist, ids=testids)
 def test_bad_stdout_specs(spec):
     """Testing bad stdout spec cases"""
@@ -54,7 +52,27 @@ def test_bad_stdout_specs(spec):
         assert False, "Did not raise expected exception"
 
 
-@pytest.mark.issue363
+@pytest.mark.issue3328
+@pytest.mark.unix_filesystem_permissions_required
+def test_bad_stdout_file():
+    """Testing bad stderr file"""
+
+    o = "/bad/dir/t2.out"
+
+    fn = echo_to_streams("Hello world", stdout=o, stderr='t.err')
+
+    try:
+        fn.result()
+    except perror.BadStdStreamFile:
+        pass
+    else:
+        assert False, "Did not raise expected exception BadStdStreamFile"
+
+    return
+
+
+@pytest.mark.issue3328
+@pytest.mark.unix_filesystem_permissions_required
 def test_bad_stderr_file():
     """Testing bad stderr file"""
 
@@ -72,8 +90,9 @@ def test_bad_stderr_file():
     return
 
 
-@pytest.mark.issue363
-def test_stdout_truncate(tmpd_cwd):
+@pytest.mark.executor_supports_std_stream_tuples
+@pytest.mark.shared_fs
+def test_stdout_truncate(tmpd_cwd, caplog):
     """Testing truncation of prior content of stdout"""
 
     out = (str(tmpd_cwd / 't1.out'), 'w')
@@ -88,9 +107,12 @@ def test_stdout_truncate(tmpd_cwd):
     assert len1 == 1
     assert len1 == len2
 
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR
 
-@pytest.mark.issue363
-def test_stdout_append(tmpd_cwd):
+
+@pytest.mark.shared_fs
+def test_stdout_append(tmpd_cwd, caplog):
     """Testing appending to prior content of stdout (default open() mode)"""
 
     out = str(tmpd_cwd / 't1.out')
@@ -103,3 +125,6 @@ def test_stdout_append(tmpd_cwd):
     len2 = len(open(out).readlines())
 
     assert len1 == 1 and len2 == 2
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR

parsl/tests/test_callables.py

@@ -5,10 +5,10 @@
 
 import importlib
 import pathlib
-import parsl
-
 from functools import partial
 
+import parsl
+
 
 @parsl.python_app
 def app(x):