parsl 2024.4.15__py3-none-any.whl → 2024.4.29__py3-none-any.whl

parsl/tests/conftest.py

@@ -3,8 +3,10 @@ import itertools
 import logging
 import os
 import pathlib
+import random
 import re
 import shutil
+import string
 import time
 import types
 import signal
@@ -139,7 +141,7 @@ def pytest_configure(config):
     )
     config.addinivalue_line(
         'markers',
-        'staging_required: Marks tests that require a staging provider, when there is no sharedFS)'
+        'staging_required: Marks tests that require a staging provider, when there is no sharedFS'
     )
     config.addinivalue_line(
         'markers',
@@ -245,6 +247,7 @@ def load_dfk_local_module(request, pytestconfig, tmpd_cwd_session):
 
         if callable(local_teardown):
             local_teardown()
+            assert DataFlowKernelLoader._dfk is None, "Expected teardown to clear DFK"
 
         if local_config:
             if parsl.dfk() != dfk:
@@ -421,3 +424,11 @@ def try_assert():
             raise AssertionError("Bad assert call: no attempts or timeout period")
 
     yield _impl
+
+
+@pytest.fixture
+def randomstring():
+    def func(length=5, alphabet=string.ascii_letters):
+        return "".join(random.choice(alphabet) for _ in range(length))
+
+    return func

parsl/tests/test_bash_apps/test_basic.py

@@ -50,6 +50,8 @@ def test_auto_log_filename_format(caplog):
         foo_future.result())
 
     log_fpath = foo_future.stdout
+    assert isinstance(log_fpath, str)
+
     log_pattern = fr".*/task_\d+_foo_{app_label}"
     assert re.match(log_pattern, log_fpath), 'Output file "{0}" does not match pattern "{1}"'.format(
         log_fpath, log_pattern)

parsl/tests/test_bash_apps/test_std_uri.py

@@ -0,0 +1,128 @@
+import logging
+import parsl
+import pytest
+import zipfile
+
+from functools import partial
+from parsl.app.futures import DataFuture
+from parsl.data_provider.files import File
+from parsl.executors import ThreadPoolExecutor
+
+
+@parsl.bash_app
+def app_stdout(stdout=parsl.AUTO_LOGNAME):
+    return "echo hello"
+
+
+def const_str(cpath, task_record, err_or_out):
+    return cpath
+
+
+def const_with_cpath(autopath_specifier, content_path, caplog):
+    with parsl.load(parsl.Config(std_autopath=partial(const_str, autopath_specifier))):
+        fut = app_stdout()
+
+        # we don't have to wait for a result to check this attributes
+        assert fut.stdout is autopath_specifier
+
+        # there is no DataFuture to wait for in the str case: the model is that
+        # the stdout will be immediately available on task completion.
+        fut.result()
+
+    with open(content_path, "r") as file:
+        assert file.readlines() == ["hello\n"]
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR
+
+    parsl.clear()
+
+
+@pytest.mark.local
+def test_std_autopath_const_str(caplog, tmpd_cwd):
+    """Tests str and tuple mode autopaths with constant autopath, which should
+    all be passed through unmodified.
+    """
+    cpath = str(tmpd_cwd / "CONST")
+    const_with_cpath(cpath, cpath, caplog)
+
+
+@pytest.mark.local
+def test_std_autopath_const_pathlike(caplog, tmpd_cwd):
+    cpath = tmpd_cwd / "CONST"
+    const_with_cpath(cpath, cpath, caplog)
+
+
+@pytest.mark.local
+def test_std_autopath_const_tuples(caplog, tmpd_cwd):
+    file = tmpd_cwd / "CONST"
+    cpath = (file, "w")
+    const_with_cpath(cpath, file, caplog)
+
+
+class URIFailError(Exception):
+    pass
+
+
+def fail_uri(task_record, err_or_out):
+    raise URIFailError("Deliberate failure in std stream filename generation")
+
+
+@pytest.mark.local
+def test_std_autopath_fail(caplog):
+    with parsl.load(parsl.Config(std_autopath=fail_uri)):
+        with pytest.raises(URIFailError):
+            app_stdout()
+
+    parsl.clear()
+
+
+@parsl.bash_app
+def app_both(stdout=parsl.AUTO_LOGNAME, stderr=parsl.AUTO_LOGNAME):
+    return "echo hello; echo goodbye >&2"
+
+
+def zip_uri(base, task_record, err_or_out):
+    """Should generate Files in base.zip like app_both.0.out or app_both.123.err"""
+    zip_path = base / "base.zip"
+    file = f"{task_record['func_name']}.{task_record['id']}.{task_record['try_id']}.{err_or_out}"
+    return File(f"zip:{zip_path}/{file}")
+
+
+@pytest.mark.local
+def test_std_autopath_zip(caplog, tmpd_cwd):
+    with parsl.load(parsl.Config(run_dir=str(tmpd_cwd),
+                                 executors=[ThreadPoolExecutor(working_dir=str(tmpd_cwd))],
+                                 std_autopath=partial(zip_uri, tmpd_cwd))):
+        futs = []
+
+        for _ in range(10):
+            fut = app_both()
+
+            # assertions that should hold after submission
+            assert isinstance(fut.stdout, DataFuture)
+            assert fut.stdout.file_obj.url.startswith("zip")
+
+            futs.append(fut)
+
+        # Barrier for all the stageouts to complete so that we can
+        # poke at the zip file.
+        [(fut.stdout.result(), fut.stderr.result()) for fut in futs]
+
+        with zipfile.ZipFile(tmpd_cwd / "base.zip") as z:
+            for fut in futs:
+
+                assert fut.done(), "AppFuture should be done if stageout is done"
+
+                stdout_relative_path = f"app_both.{fut.tid}.0.stdout"
+                with z.open(stdout_relative_path) as f:
+                    assert f.readlines() == [b'hello\n']
+
+                stderr_relative_path = f"app_both.{fut.tid}.0.stderr"
+                with z.open(stderr_relative_path) as f:
+                    assert f.readlines()[-1] == b'goodbye\n'
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR
+
+    parsl.clear()

parsl/tests/test_checkpointing/test_periodic.py

@@ -1,16 +1,12 @@
-import argparse
-import time
-
 import pytest
 
 import parsl
 from parsl.app.app import python_app
-from parsl.tests.configs.local_threads_checkpoint_periodic import config
+from parsl.tests.configs.local_threads_checkpoint_periodic import fresh_config
 
 
 def local_setup():
-    global dfk
-    dfk = parsl.load(config)
+    parsl.load(fresh_config())
 
 
 def local_teardown():
@@ -27,40 +23,31 @@ def slow_double(x, sleep_dur=1):
 
 
 def tstamp_to_seconds(line):
-    print("Parsing line: ", line)
     f = line.partition(" ")[0]
     return float(f)
 
 
 @pytest.mark.local
-def test_periodic(n=4):
+def test_periodic():
     """Test checkpointing with task_periodic behavior
     """
-
-    d = {}
-
-    print("Launching : ", n)
-    for i in range(0, n):
-        d[i] = slow_double(i)
-    print("Done launching")
-
-    for i in range(0, n):
-        d[i].result()
-    print("Done sleeping")
-
-    time.sleep(16)
-    dfk.cleanup()
+    h, m, s = map(int, parsl.dfk().config.checkpoint_period.split(":"))
+    assert h == 0, "Verify test setup"
+    assert m == 0, "Verify test setup"
+    assert s > 0, "Verify test setup"
+    sleep_for = s + 1
+    with parsl.dfk():
+        futs = [slow_double(sleep_for) for _ in range(4)]
+        [f.result() for f in futs]
 
     # Here we will check if the loglines came back with 5 seconds deltas
-    print("Rundir: ", dfk.run_dir)
-
-    with open("{}/parsl.log".format(dfk.run_dir), 'r') as f:
+    with open("{}/parsl.log".format(parsl.dfk().run_dir)) as f:
         log_lines = f.readlines()
-        expected_msg = " Done checkpointing"
-        expected_msg2 = " No tasks checkpointed in this pass"
-
-        lines = [line for line in log_lines if expected_msg in line or expected_msg2 in line]
-        assert len(lines) >= 3, "Insufficient checkpoint lines in logfile"
-        deltas = [tstamp_to_seconds(line) for line in lines]
-        assert deltas[1] - deltas[0] < 5.5, "Delta between checkpoints exceeded period"
-        assert deltas[2] - deltas[1] < 5.5, "Delta between checkpoints exceeded period"
+    expected_msg = " Done checkpointing"
+    expected_msg2 = " No tasks checkpointed in this pass"
+
+    lines = [line for line in log_lines if expected_msg in line or expected_msg2 in line]
+    assert len(lines) >= 3, "Insufficient checkpoint lines in logfile"
+    deltas = [tstamp_to_seconds(line) for line in lines]
+    assert deltas[1] - deltas[0] < 5.5, "Delta between checkpoints exceeded period"
+    assert deltas[2] - deltas[1] < 5.5, "Delta between checkpoints exceeded period"

parsl/tests/test_checkpointing/test_task_exit.py

@@ -15,7 +15,7 @@ def local_setup():
 
 
 def local_teardown():
-    parsl.dfk().cleanup
+    parsl.dfk().cleanup()
     parsl.clear()
 
 

parsl/tests/test_htex/test_basic.py

@@ -23,6 +23,6 @@ def dummy():
 
 
 @pytest.mark.local
-def test_that_it_fails():
+def test_app():
     x = dummy()
-    x.result()
+    assert x.result() is None

parsl/tests/test_htex/test_missing_worker.py

@@ -37,7 +37,3 @@ def test_that_it_fails():
         raise Exception("The app somehow ran without a valid worker")
 
     assert parsl.dfk().config.executors[0]._executor_bad_state.is_set()
-
-    # htex needs shutting down explicitly because dfk.cleanup() will not
-    # do that, as it is in bad state
-    parsl.dfk().config.executors[0].shutdown()

parsl/tests/test_htex/test_zmq_binding.py

@@ -53,6 +53,7 @@ def test_interchange_binding_with_address(cert_dir: Optional[str]):
     assert ix.interchange_address == address
 
 
+@pytest.mark.skip("This behaviour is possibly unexpected. See issue #3037")
 @pytest.mark.local
 @pytest.mark.parametrize("encrypted", (True, False), indirect=True)
 def test_interchange_binding_with_non_ipv4_address(cert_dir: Optional[str]):

parsl/tests/test_monitoring/test_stdouterr.py

@@ -0,0 +1,137 @@
+"""Tests monitoring records app name under various decoration patterns.
+"""
+
+import logging
+import os
+import parsl
+import pytest
+import re
+import time
+
+from typing import Union
+
+from parsl.config import Config
+from parsl.data_provider.files import File
+from parsl.data_provider.data_manager import default_staging
+from parsl.data_provider.staging import Staging
+from parsl.executors import HighThroughputExecutor
+from parsl.monitoring import MonitoringHub
+from parsl.providers import LocalProvider
+
+
+def fresh_config(run_dir):
+    return Config(
+        run_dir=str(run_dir),
+        executors=[
+            HighThroughputExecutor(
+                address="127.0.0.1",
+                label="htex_Local",
+                provider=LocalProvider(
+                    init_blocks=1,
+                    min_blocks=1,
+                    max_blocks=1,
+                )
+            )
+        ],
+        strategy='simple',
+        strategy_period=0.1,
+        monitoring=MonitoringHub(
+                        hub_address="localhost",
+                        hub_port=55055,
+        )
+    )
+
+
+@parsl.python_app
+def stdapp(stdout=None, stderr=None):
+    pass
+
+
+class ArbitraryPathLike(os.PathLike):
+    def __init__(self, path: Union[str, bytes]) -> None:
+        self.path = path
+
+    def __fspath__(self) -> Union[str, bytes]:
+        return self.path
+
+
+class ArbitraryStaging(Staging):
+    """This staging provider will not actually do any staging, but will
+    accept arbitrary: scheme URLs. That's enough for this monitoring test
+    which doesn't need any actual stage out action to happen.
+    """
+    def can_stage_out(self, file):
+        return file.scheme == "arbitrary"
+
+
+@pytest.mark.local
+@pytest.mark.parametrize('stdx,expected_stdx',
+                         [('hello.txt', 'hello.txt'),
+                          (None, ''),
+                          (('tuple.txt', 'w'), 'tuple.txt'),
+                          (ArbitraryPathLike('pl.txt'), 'pl.txt'),
+                          (ArbitraryPathLike(b'pl2.txt'), 'pl2.txt'),
+                          ((ArbitraryPathLike('pl3.txt'), 'w'), 'pl3.txt'),
+                          ((ArbitraryPathLike(b'pl4.txt'), 'w'), 'pl4.txt'),
+                          (parsl.AUTO_LOGNAME,
+                              lambda p:
+                              isinstance(p, str) and
+                              os.path.isabs(p) and
+                              re.match("^.*/task_0000_stdapp\\.std...$", p)),
+                          (File("arbitrary:abc123"), "arbitrary:abc123"),
+                          (File("file:///tmp/pl5"), "file:///tmp/pl5"),
+                          ])
+@pytest.mark.parametrize('stream', ['stdout', 'stderr'])
+def test_stdstream_to_monitoring(stdx, expected_stdx, stream, tmpd_cwd, caplog):
+    """This tests that various forms of stdout/err specification are
+       represented in monitoring correctly. The stderr and stdout codepaths
+       are generally duplicated, rather than factorised, and so this test
+       runs the same tests on both stdout and stderr.
+    """
+
+    # this is imported here rather than at module level because
+    # it isn't available in a plain parsl install, so this module
+    # would otherwise fail to import and break even a basic test
+    # run.
+    import sqlalchemy
+
+    c = fresh_config(tmpd_cwd)
+    c.monitoring.logging_endpoint = f"sqlite:///{tmpd_cwd}/monitoring.db"
+    c.executors[0].storage_access = default_staging + [ArbitraryStaging()]
+
+    with parsl.load(c):
+        kwargs = {stream: stdx}
+        stdapp(**kwargs).result()
+
+    parsl.clear()
+
+    engine = sqlalchemy.create_engine(c.monitoring.logging_endpoint)
+    with engine.begin() as connection:
+
+        def count_rows(table: str):
+            result = connection.execute(f"SELECT COUNT(*) FROM {table}")
+            (c, ) = result.first()
+            return c
+
+        # one workflow...
+        assert count_rows("workflow") == 1
+
+        # ... with one task ...
+        assert count_rows("task") == 1
+
+        # ... that was tried once ...
+        assert count_rows("try") == 1
+
+        # ... and has the expected name.
+        result = connection.execute(f"SELECT task_{stream} FROM task")
+        (c, ) = result.first()
+
+        if isinstance(expected_stdx, str):
+            assert c == expected_stdx
+        elif callable(expected_stdx):
+            assert expected_stdx(c)
+        else:
+            raise RuntimeError("Bad expected_stdx value")
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR

parsl/tests/test_mpi_apps/test_resource_spec.py

@@ -25,17 +25,11 @@ from parsl.executors.high_throughput.mpi_prefix_composer import (
 EXECUTOR_LABEL = "MPI_TEST"
 
 
-def local_setup():
+def local_config():
     config = fresh_config()
     config.executors[0].label = EXECUTOR_LABEL
     config.executors[0].max_workers_per_node = 1
-    parsl.load(config)
-
-
-def local_teardown():
-    logging.warning("Exiting")
-    parsl.dfk().cleanup()
-    parsl.clear()
+    return config
 
 
 @python_app

parsl/tests/test_python_apps/test_context_manager.py

@@ -24,15 +24,15 @@ def local_teardown():
 
 
 @pytest.mark.local
-def test_within_context_manger():
+def test_within_context_manger(tmpd_cwd):
     config = fresh_config()
     with parsl.load(config=config) as dfk:
         assert isinstance(dfk, DataFlowKernel)
 
-        bash_future = foo(1)
+        bash_future = foo(1, stdout=tmpd_cwd / 'foo.stdout')
         assert bash_future.result() == 0
 
-        with open('foo.stdout', 'r') as f:
+        with open(tmpd_cwd / 'foo.stdout', 'r') as f:
             assert f.read() == "2\n"
 
     with pytest.raises(NoDataFlowKernelError) as excinfo:

parsl/tests/test_scaling/test_regression_1621.py

@@ -9,6 +9,14 @@ from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
 from parsl.providers import LocalProvider
 
+# Timing notes:
+# The configured strategy_period must be much smaller than the delay in
+# app() so that multiple iterations of the strategy have had a chance
+# to (mis)behave.
+# The status polling interval in OneShotLocalProvider must be much bigger
+# than the above times, so that the job status cached from the provider
+# will not be updated while the single invocation of app() runs.
+
 
 @parsl.python_app
 def app():
@@ -55,20 +63,12 @@ def test_one_block(tmpd_cwd):
             )
         ],
         strategy='simple',
+        strategy_period=0.1
     )
 
-    parsl.load(config)
-    dfk = parsl.dfk()
-
-    def poller():
-        import time
-        while True:
-            dfk.job_status_poller.poll()
-            time.sleep(0.1)
+    with parsl.load(config):
+        app().result()
 
-    threading.Thread(target=poller, daemon=True).start()
-    app().result()
-    parsl.dfk().cleanup()
     parsl.clear()
 
     assert oneshot_provider.recorded_submits == 1

parsl/tests/test_staging/test_staging_stdout.py

@@ -0,0 +1,61 @@
+import logging
+import os
+import parsl
+import pytest
+import zipfile
+
+from parsl.app.futures import DataFuture
+from parsl.tests.configs.htex_local import fresh_config as local_config
+from parsl.data_provider.files import File
+
+
+@parsl.bash_app
+def output_to_stds(*, stdout=parsl.AUTO_LOGNAME, stderr=parsl.AUTO_LOGNAME):
+    return "echo hello ; echo goodbye >&2"
+
+
+def test_stdout_staging_file(tmpd_cwd, caplog):
+    basename = str(tmpd_cwd) + "/stdout.txt"
+    stdout_file = File("file://" + basename)
+
+    app_future = output_to_stds(stdout=stdout_file)
+
+    assert isinstance(app_future.stdout, DataFuture)
+    app_future.stdout.result()
+
+    assert os.path.exists(basename)
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR
+
+
+def test_stdout_stderr_staging_zip(tmpd_cwd, caplog):
+    zipfile_name = str(tmpd_cwd) + "/staging.zip"
+    stdout_relative_path = "somewhere/test-out.txt"
+    stdout_file = File("zip:" + zipfile_name + "/" + stdout_relative_path)
+
+    stderr_relative_path = "somewhere/test-error.txt"
+    stderr_file = File("zip:" + zipfile_name + "/" + stderr_relative_path)
+
+    app_future = output_to_stds(stdout=stdout_file, stderr=stderr_file)
+
+    assert isinstance(app_future.stdout, DataFuture)
+    app_future.stdout.result()
+
+    # check the file exists as soon as possible
+    assert os.path.exists(zipfile_name)
+    with zipfile.ZipFile(zipfile_name) as z:
+        with z.open(stdout_relative_path) as f:
+            assert f.readlines() == [b'hello\n']
+
+    assert isinstance(app_future.stderr, DataFuture)
+    app_future.stderr.result()
+    with zipfile.ZipFile(zipfile_name) as z:
+        with z.open(stderr_relative_path) as f:
+            # The last line of stderr should be goodbye, but Parsl will write
+            # other Parsl-specific into to stderr before that, so only assert
+            # the behaviour of the final line.
+            assert f.readlines()[-1] == b'goodbye\n'
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR

parsl/tests/test_staging/test_zip_in.py

@@ -0,0 +1,42 @@
+import parsl
+import pytest
+import random
+import zipfile
+
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+
+from parsl.providers import LocalProvider
+from parsl.channels import LocalChannel
+from parsl.launchers import SimpleLauncher
+
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@parsl.python_app
+def count_lines(file):
+    with open(file, "r") as f:
+        return len(f.readlines())
+
+
+@pytest.mark.local
+def test_zip_in(tmpd_cwd):
+    # basic test of zip file stage-in
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    zip_file = File(f"zip:{zip_path / file_base}")
+
+    # create a zip file containing one file with some abitrary number of lines
+    n_lines = random.randint(0, 1000)
+
+    with zipfile.ZipFile(zip_path, mode='w') as z:
+        with z.open(file_base, mode='w') as f:
+            for _ in range(n_lines):
+                f.write(b'someline\n')
+
+    app_future = count_lines(zip_file)
+
+    assert app_future.result() == n_lines

parsl/tests/test_staging/test_zip_to_zip.py

@@ -0,0 +1,44 @@
+import parsl
+import pytest
+import random
+import zipfile
+
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+
+from parsl.providers import LocalProvider
+from parsl.channels import LocalChannel
+from parsl.launchers import SimpleLauncher
+
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@parsl.python_app
+def generate_lines(n: int, *, outputs):
+    with open(outputs[0], "w") as f:
+        for x in range(n):
+            # write numbered lines
+            f.write(str(x) + "\n")
+
+
+@parsl.python_app
+def count_lines(file):
+    with open(file, "r") as f:
+        return len(f.readlines())
+
+
+@pytest.mark.local
+def test_zip_pipeline(tmpd_cwd):
+    # basic test of zip file stage-in
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    zip_file = File(f"zip:{zip_path / file_base}")
+
+    n_lines = random.randint(0, 1000)
+    generate_fut = generate_lines(n_lines, outputs=[zip_file])
+    n_lines_out = count_lines(generate_fut.outputs[0]).result()
+
+    assert n_lines == n_lines_out