parsl 2024.4.1__py3-none-any.whl → 2024.4.15__py3-none-any.whl

parsl/tests/test_bash_apps/test_memoize.py

@@ -12,7 +12,6 @@ def fail_on_presence(outputs=()):
 # This test is an oddity that requires a shared-FS and simply
 # won't work if there's a staging provider.
 # @pytest.mark.sharedFS_required
-@pytest.mark.issue363
 def test_bash_memoization(tmpd_cwd, n=2):
     """Testing bash memoization
     """
@@ -33,7 +32,6 @@ def fail_on_presence_kw(outputs=(), foo=None):
 # This test is an oddity that requires a shared-FS and simply
 # won't work if there's a staging provider.
 # @pytest.mark.sharedFS_required
-@pytest.mark.issue363
 def test_bash_memoization_keywords(tmpd_cwd, n=2):
     """Testing bash memoization
     """

parsl/tests/test_bash_apps/test_memoize_ignore_args.py

@@ -22,7 +22,6 @@ def no_checkpoint_stdout_app_ignore_args(stdout=None):
     return "echo X"
 
 
-@pytest.mark.issue363
 def test_memo_stdout():
 
     # this should run and create a file named after path_x

parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py

@@ -30,7 +30,6 @@ def no_checkpoint_stdout_app(stdout=None):
     return "echo X"
 
 
-@pytest.mark.issue363
 def test_memo_stdout():
 
     assert const_list_x == const_list_x_arg

parsl/tests/test_bash_apps/test_multiline.py

@@ -14,7 +14,6 @@ def multiline(inputs=(), outputs=(), stderr=None, stdout=None):
     """.format(inputs=inputs, outputs=outputs)
 
 
-@pytest.mark.issue363
 def test_multiline(tmpd_cwd):
     so, se = tmpd_cwd / "std.out", tmpd_cwd / "std.err"
     f = multiline(

parsl/tests/test_bash_apps/test_stdout.py

@@ -1,3 +1,4 @@
+import logging
 import os
 
 import pytest
@@ -35,7 +36,6 @@ testids = [
 ]
 
 
-@pytest.mark.issue363
 @pytest.mark.parametrize('spec', speclist, ids=testids)
 def test_bad_stdout_specs(spec):
     """Testing bad stdout spec cases"""
@@ -54,7 +54,7 @@ def test_bad_stdout_specs(spec):
         assert False, "Did not raise expected exception"
 
 
-@pytest.mark.issue363
+@pytest.mark.issue3328
 def test_bad_stderr_file():
     """Testing bad stderr file"""
 
@@ -72,8 +72,8 @@ def test_bad_stderr_file():
     return
 
 
-@pytest.mark.issue363
-def test_stdout_truncate(tmpd_cwd):
+@pytest.mark.executor_supports_std_stream_tuples
+def test_stdout_truncate(tmpd_cwd, caplog):
     """Testing truncation of prior content of stdout"""
 
     out = (str(tmpd_cwd / 't1.out'), 'w')
@@ -88,9 +88,11 @@ def test_stdout_truncate(tmpd_cwd):
     assert len1 == 1
     assert len1 == len2
 
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR
 
-@pytest.mark.issue363
-def test_stdout_append(tmpd_cwd):
+
+def test_stdout_append(tmpd_cwd, caplog):
     """Testing appending to prior content of stdout (default open() mode)"""
 
     out = str(tmpd_cwd / 't1.out')
@@ -103,3 +105,6 @@ def test_stdout_append(tmpd_cwd):
     len2 = len(open(out).readlines())
 
     assert len1 == 1 and len2 == 2
+
+    for record in caplog.records:
+        assert record.levelno < logging.ERROR

parsl/tests/test_monitoring/test_basic.py

@@ -1,10 +1,13 @@
-import logging
 import os
 import parsl
 import pytest
 import time
 
-logger = logging.getLogger(__name__)
+from parsl import HighThroughputExecutor
+from parsl.config import Config
+from parsl.executors.taskvine import TaskVineExecutor
+from parsl.executors.taskvine import TaskVineManagerConfig
+from parsl.monitoring import MonitoringHub
 
 
 @parsl.python_app
@@ -18,34 +21,56 @@ def this_app():
     return 5
 
 
+# The below fresh configs are for use in parametrization, and should return
+# a configuration that is suitably configured for monitoring.
+
+def htex_config():
+    from parsl.tests.configs.htex_local_alternate import fresh_config
+    return fresh_config()
+
+
+def workqueue_config():
+    from parsl.tests.configs.workqueue_ex import fresh_config
+    c = fresh_config()
+    c.monitoring = MonitoringHub(
+                        hub_address="localhost",
+                        resource_monitoring_interval=1)
+    return c
+
+
+def taskvine_config():
+    c = Config(executors=[TaskVineExecutor(manager_config=TaskVineManagerConfig(port=9000),
+                                           worker_launch_method='provider')],
+
+               monitoring=MonitoringHub(hub_address="localhost",
+                                        resource_monitoring_interval=1))
+    return c
+
+
 @pytest.mark.local
-def test_row_counts():
+@pytest.mark.parametrize("fresh_config", [htex_config, workqueue_config, taskvine_config])
+def test_row_counts(tmpd_cwd, fresh_config):
     # this is imported here rather than at module level because
     # it isn't available in a plain parsl install, so this module
     # would otherwise fail to import and break even a basic test
     # run.
     import sqlalchemy
     from sqlalchemy import text
-    from parsl.tests.configs.htex_local_alternate import fresh_config
 
-    if os.path.exists("runinfo/monitoring.db"):
-        logger.info("Monitoring database already exists - deleting")
-        os.remove("runinfo/monitoring.db")
+    db_url = f"sqlite:///{tmpd_cwd}/monitoring.db"
 
-    logger.info("loading parsl")
-    parsl.load(fresh_config())
+    config = fresh_config()
+    config.run_dir = tmpd_cwd
+    config.monitoring.logging_endpoint = db_url
 
-    logger.info("invoking and waiting for result")
-    assert this_app().result() == 5
+    with parsl.load(config):
+        assert this_app().result() == 5
 
-    logger.info("cleaning up parsl")
-    parsl.dfk().cleanup()
     parsl.clear()
 
     # at this point, we should find one row in the monitoring database.
 
-    logger.info("checking database content")
-    engine = sqlalchemy.create_engine("sqlite:///runinfo/monitoring.db")
+    engine = sqlalchemy.create_engine(db_url)
     with engine.begin() as connection:
 
         result = connection.execute(text("SELECT COUNT(*) FROM workflow"))
@@ -67,10 +92,12 @@ def test_row_counts():
         (c, ) = result.first()
         assert c == 0
 
-        # Two entries: one showing manager active, one inactive
-        result = connection.execute(text("SELECT COUNT(*) FROM node"))
-        (c, ) = result.first()
-        assert c == 2
+        if isinstance(config.executors[0], HighThroughputExecutor):
+            # The node table is specific to the HighThroughputExecutor
+            # Two entries: one showing manager active, one inactive
+            result = connection.execute(text("SELECT COUNT(*) FROM node"))
+            (c, ) = result.first()
+            assert c == 2
 
         # There should be one block polling status
         # local provider has a status_polling_interval of 5s
@@ -81,5 +108,3 @@ def test_row_counts():
         result = connection.execute(text("SELECT COUNT(*) FROM resource"))
         (c, ) = result.first()
         assert c >= 1
-
-    logger.info("all done")

parsl/tests/test_monitoring/test_fuzz_zmq.py

@@ -4,6 +4,7 @@ import parsl
 import pytest
 import socket
 import time
+import zmq
 
 logger = logging.getLogger(__name__)
 
@@ -48,8 +49,16 @@ def test_row_counts():
         s.connect((hub_address, hub_zmq_port))
         s.sendall(b'fuzzing\r')
 
+    context = zmq.Context()
+    channel_timeout = 10000  # in milliseconds
+    hub_channel = context.socket(zmq.DEALER)
+    hub_channel.setsockopt(zmq.LINGER, 0)
+    hub_channel.set_hwm(0)
+    hub_channel.setsockopt(zmq.SNDTIMEO, channel_timeout)
+    hub_channel.connect("tcp://{}:{}".format(hub_address, hub_zmq_port))
+
     # this will send a non-object down the DFK's existing ZMQ connection
-    parsl.dfk().monitoring._dfk_channel.send(b'FuzzyByte\rSTREAM')
+    hub_channel.send(b'FuzzyByte\rSTREAM')
 
     # This following attack is commented out, because monitoring is not resilient
     # to this.

parsl/tests/test_python_apps/test_outputs.py

@@ -16,7 +16,6 @@ def double(x, outputs=[]):
 whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs', '*threads*')
 
 
-@pytest.mark.issue363
 def test_launch_apps(tmpd_cwd, n=2):
     outdir = tmpd_cwd / "outputs"
     outdir.mkdir()

parsl/tests/test_scaling/test_scale_down_htex_unregistered.py

@@ -0,0 +1,74 @@
+import logging
+import time
+
+import pytest
+
+import parsl
+
+from parsl import File, python_app
+from parsl.jobs.states import JobState, TERMINAL_STATES
+from parsl.providers import LocalProvider
+from parsl.channels import LocalChannel
+from parsl.launchers import SingleNodeLauncher
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+
+logger = logging.getLogger(__name__)
+
+_max_blocks = 1
+_min_blocks = 0
+
+
+def local_config():
+    return Config(
+        executors=[
+            HighThroughputExecutor(
+                heartbeat_period=1,
+                heartbeat_threshold=2,
+                poll_period=100,
+                label="htex_local",
+                address="127.0.0.1",
+                max_workers=1,
+                encrypted=True,
+                launch_cmd="sleep inf",
+                provider=LocalProvider(
+                    channel=LocalChannel(),
+                    init_blocks=1,
+                    max_blocks=_max_blocks,
+                    min_blocks=_min_blocks,
+                    launcher=SingleNodeLauncher(),
+                ),
+            )
+        ],
+        max_idletime=0.5,
+        strategy='htex_auto_scale',
+        strategy_period=0.1
+    )
+
+
+# see issue #1885 for details of failures of this test.
+# at the time of issue #1885 this test was failing frequently
+# in CI.
+@pytest.mark.local
+def test_scaledown_with_register(try_assert):
+    dfk = parsl.dfk()
+    htex = dfk.executors['htex_local']
+
+    num_managers = len(htex.connected_managers())
+    assert num_managers == 0, "Expected 0 managers at start"
+
+    try_assert(lambda: len(htex.status()),
+               fail_msg="Expected 1 block at start")
+
+    s = htex.status()
+    assert s['0'].state == JobState.RUNNING, "Expected block to be in RUNNING"
+
+    def check_zero_blocks():
+        s = htex.status()
+        return len(s) == 1 and s['0'].state in TERMINAL_STATES
+
+    try_assert(
+        check_zero_blocks,
+        fail_msg="Expected 0 blocks after idle scaledown",
+        timeout_ms=15000,
+    )

parsl/tests/test_staging/test_zip_out.py

@@ -0,0 +1,113 @@
+import parsl
+import pytest
+import zipfile
+
+from parsl.data_provider.files import File
+from parsl.data_provider.data_manager import default_staging
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+
+from parsl.providers import LocalProvider
+from parsl.channels import LocalChannel
+from parsl.launchers import SimpleLauncher
+
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+
+from parsl.tests.configs.htex_local import fresh_config as local_config
+
+
+@pytest.mark.local
+def test_zip_path_split():
+    from parsl.data_provider.zip import zip_path_split
+    assert zip_path_split("/tmp/foo/this.zip/inside/here.txt") == ("/tmp/foo/this.zip", "inside/here.txt")
+
+
+@parsl.bash_app
+def output_something(outputs=()):
+    """This should output something into every specified output file:
+    the position in the output sequence will be written into the
+    corresponding output file.
+    """
+    cmds = []
+    for n in range(len(outputs)):
+        cmds.append(f"echo {n} > {outputs[n]}")
+
+    return "; ".join(cmds)
+
+
+@pytest.mark.local
+def test_zip_out(tmpd_cwd):
+    # basic test of zip file stage-out
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    of = File(f"zip:{zip_path / file_base}")
+
+    app_future = output_something(outputs=[of])
+    output_file_future = app_future.outputs[0]
+
+    app_future.result()
+    output_file_future.result()
+
+    assert zipfile.is_zipfile(zip_path)
+
+    with zipfile.ZipFile(zip_path) as z:
+        assert file_base in z.namelist()
+        assert len(z.namelist()) == 1
+        with z.open(file_base) as f:
+            assert f.readlines() == [b'0\n']
+
+
+@pytest.mark.local
+def test_zip_out_multi(tmpd_cwd):
+    # tests multiple files, multiple zip files and multiple
+    # sub-paths
+
+    zip_path_1 = tmpd_cwd / "container1.zip"
+    zip_path_2 = tmpd_cwd / "container2.zip"
+
+    relative_file_path_1 = "a/b/c/data.txt"
+    relative_file_path_2 = "something.txt"
+    relative_file_path_3 = "a/d/other.txt"
+    of1 = File(f"zip:{zip_path_1 / relative_file_path_1}")
+    of2 = File(f"zip:{zip_path_1 / relative_file_path_2}")
+    of3 = File(f"zip:{zip_path_2 / relative_file_path_3}")
+
+    app_future = output_something(outputs=[of1, of2, of3])
+
+    for f in app_future.outputs:
+        f.result()
+
+    app_future.result()
+
+    assert zipfile.is_zipfile(zip_path_1)
+
+    with zipfile.ZipFile(zip_path_1) as z:
+        assert relative_file_path_1 in z.namelist()
+        assert relative_file_path_2 in z.namelist()
+        assert len(z.namelist()) == 2
+        with z.open(relative_file_path_1) as f:
+            assert f.readlines() == [b'0\n']
+        with z.open(relative_file_path_2) as f:
+            assert f.readlines() == [b'1\n']
+
+    assert zipfile.is_zipfile(zip_path_2)
+
+    with zipfile.ZipFile(zip_path_2) as z:
+        assert relative_file_path_3 in z.namelist()
+        assert len(z.namelist()) == 1
+        with z.open(relative_file_path_3) as f:
+            assert f.readlines() == [b'2\n']
+
+
+@pytest.mark.local
+def test_zip_bad_authority(tmpd_cwd):
+    # tests that there's an exception when staging a ZIP url with an authority
+    # section specified, rather than silently ignoring it. This simulates a
+    # user who misunderstands what that piece of what a zip: URL means.
+
+    zip_path = tmpd_cwd / "container.zip"
+    file_base = "data.txt"
+    of = File(f"zip://someauthority/{zip_path / file_base}")
+
+    with pytest.raises(ZipAuthorityError):
+        output_something(outputs=[of])

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.04.01'
+VERSION = '2024.04.15'

{parsl-2024.4.1.data → parsl-2024.4.15.data}/scripts/process_worker_pool.py

@@ -361,7 +361,9 @@ class Manager:
                     kill_event.set()
                 else:
                     task_recv_counter += len(tasks)
-                    logger.debug("Got executor tasks: {}, cumulative count of tasks: {}".format([t['task_id'] for t in tasks], task_recv_counter))
+                    logger.debug("Got executor tasks: {}, cumulative count of tasks: {}".format(
+                        [t['task_id'] for t in tasks], task_recv_counter
+                    ))
 
                     for task in tasks:
                         self.task_scheduler.put_task(task)

{parsl-2024.4.1.dist-info → parsl-2024.4.15.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.4.1
+Version: 2024.4.15
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.04.01.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.04.15.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -28,6 +28,7 @@ Requires-Dist: requests
 Requires-Dist: paramiko
 Requires-Dist: psutil >=5.5.1
 Requires-Dist: setproctitle
+Requires-Dist: filelock <4,>=3.13
 Provides-Extra: all
 Requires-Dist: sqlalchemy <2,>=1.4 ; extra == 'all'
 Requires-Dist: pydot ; extra == 'all'