parsl 2024.10.21__py3-none-any.whl → 2024.11.4__py3-none-any.whl

parsl/tests/test_bash_apps/test_memoize.py

@@ -9,9 +9,7 @@ def fail_on_presence(outputs=()):
     return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])
 
 
-# This test is an oddity that requires a shared-FS and simply
-# won't work if there's a staging provider.
-# @pytest.mark.sharedFS_required
+@pytest.mark.shared_fs
 def test_bash_memoization(tmpd_cwd, n=2):
     """Testing bash memoization
     """
@@ -29,9 +27,7 @@ def fail_on_presence_kw(outputs=(), foo=None):
     return 'if [ -f {0} ] ; then exit 1 ; else touch {0}; fi'.format(outputs[0])
 
 
-# This test is an oddity that requires a shared-FS and simply
-# won't work if there's a staging provider.
-# @pytest.mark.sharedFS_required
+@pytest.mark.shared_fs
 def test_bash_memoization_keywords(tmpd_cwd, n=2):
     """Testing bash memoization
     """

parsl/tests/test_bash_apps/test_memoize_ignore_args.py

@@ -1,5 +1,7 @@
 import os
 
+import pytest
+
 import parsl
 from parsl.app.app import bash_app
 
@@ -21,6 +23,7 @@ def no_checkpoint_stdout_app_ignore_args(stdout=None):
     return "echo X"
 
 
+@pytest.mark.shared_fs
 def test_memo_stdout(tmpd_cwd):
     path_x = tmpd_cwd / "test.memo.stdout.x"
 

parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py

@@ -29,6 +29,7 @@ def no_checkpoint_stdout_app(stdout=None):
     return "echo X"
 
 
+@pytest.mark.shared_fs
 def test_memo_stdout(tmpd_cwd):
     assert const_list_x == const_list_x_arg
 

parsl/tests/test_bash_apps/test_multiline.py

@@ -14,6 +14,7 @@ def multiline(inputs=(), outputs=(), stderr=None, stdout=None):
     """.format(inputs=inputs, outputs=outputs)
 
 
+@pytest.mark.shared_fs
 def test_multiline(tmpd_cwd):
     so, se = tmpd_cwd / "std.out", tmpd_cwd / "std.err"
     f = multiline(

parsl/tests/test_bash_apps/test_stdout.py

@@ -91,6 +91,7 @@ def test_bad_stderr_file():
 
 
 @pytest.mark.executor_supports_std_stream_tuples
+@pytest.mark.shared_fs
 def test_stdout_truncate(tmpd_cwd, caplog):
     """Testing truncation of prior content of stdout"""
 
@@ -110,6 +111,7 @@ def test_stdout_truncate(tmpd_cwd, caplog):
         assert record.levelno < logging.ERROR
 
 
+@pytest.mark.shared_fs
 def test_stdout_append(tmpd_cwd, caplog):
     """Testing appending to prior content of stdout (default open() mode)"""
 

parsl/tests/{integration/test_channels → test_channels}/test_local_channel.py

@@ -1,6 +1,9 @@
+import pytest
+
 from parsl.channels.local.local import LocalChannel
 
 
+@pytest.mark.local
 def test_env():
     ''' Regression testing for issue #27
     '''
@@ -15,9 +18,8 @@ def test_env():
     x = [s for s in stdout if s.startswith("HOME=")]
     assert x, "HOME not found"
 
-    print("RC:{} \nSTDOUT:{} \nSTDERR:{}".format(rc, stdout, stderr))
-
 
+@pytest.mark.local
 def test_env_mod():
     ''' Testing for env update at execute time.
     '''
@@ -34,9 +36,3 @@ def test_env_mod():
 
     x = [s for s in stdout if s.startswith("TEST_ENV=fooo")]
     assert x, "User set env missing"
-
-
-if __name__ == "__main__":
-
-    test_env()
-    test_env_mod()

parsl/tests/test_docs/test_from_slides.py

@@ -1,5 +1,7 @@
 import os
 
+import pytest
+
 from parsl.app.app import bash_app, python_app
 from parsl.data_provider.files import File
 
@@ -15,6 +17,7 @@ def cat(inputs=[]):
         return f.readlines()
 
 
+@pytest.mark.staging_required
 def test_slides():
     """Testing code snippet from slides """
 

parsl/tests/test_docs/test_kwargs.py

@@ -1,6 +1,8 @@
 """Functions used to explain kwargs"""
 from pathlib import Path
 
+import pytest
+
 from parsl import File, python_app
 
 
@@ -19,6 +21,7 @@ def test_inputs():
     assert reduce_future.result() == 6
 
 
+@pytest.mark.shared_fs
 def test_outputs(tmpd_cwd):
     @python_app()
     def write_app(message, outputs=()):

parsl/tests/test_monitoring/test_basic.py

@@ -42,6 +42,18 @@ def htex_udp_config():
     return c
 
 
+def htex_filesystem_config():
+    """This config will force filesystem radio"""
+    from parsl.tests.configs.htex_local_alternate import fresh_config
+    c = fresh_config()
+    assert len(c.executors) == 1
+
+    assert c.executors[0].radio_mode == "htex", "precondition: htex has a radio mode attribute, configured for htex radio"
+    c.executors[0].radio_mode = "filesystem"
+
+    return c
+
+
 def workqueue_config():
     from parsl.tests.configs.workqueue_ex import fresh_config
     c = fresh_config()
@@ -61,7 +73,7 @@ def taskvine_config():
 
 
 @pytest.mark.local
-@pytest.mark.parametrize("fresh_config", [htex_config, htex_udp_config, workqueue_config, taskvine_config])
+@pytest.mark.parametrize("fresh_config", [htex_config, htex_filesystem_config, htex_udp_config, workqueue_config, taskvine_config])
 def test_row_counts(tmpd_cwd, fresh_config):
     # this is imported here rather than at module level because
     # it isn't available in a plain parsl install, so this module

parsl/tests/test_python_apps/test_outputs.py

@@ -16,6 +16,7 @@ def double(x, outputs=[]):
 whitelist = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'configs', '*threads*')
 
 
+@pytest.mark.shared_fs
 def test_launch_apps(tmpd_cwd, n=2):
     outdir = tmpd_cwd / "outputs"
     outdir.mkdir()

parsl/tests/test_regression/test_226.py

@@ -53,6 +53,7 @@ def test_get_dataframe():
     assert res.equals(data), 'Unexpected dataframe'
 
 
+@pytest.mark.shared_fs
 def test_bash_default_arg():
     if os.path.exists('std.out'):
         os.remove('std.out')

parsl/tests/test_scaling/test_worker_interchange_bad_messages_3262.py

@@ -0,0 +1,92 @@
+import os
+import signal
+import time
+
+import pytest
+import zmq
+
+import parsl
+from parsl.channels import LocalChannel
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+
+T_s = 1
+
+
+def fresh_config():
+    htex = HighThroughputExecutor(
+               heartbeat_period=1 * T_s,
+               heartbeat_threshold=3 * T_s,
+               label="htex_local",
+               worker_debug=True,
+               cores_per_worker=1,
+               encrypted=False,
+               provider=LocalProvider(
+                   channel=LocalChannel(),
+                   init_blocks=0,
+                   min_blocks=0,
+                   max_blocks=0,
+                   launcher=SimpleLauncher(),
+               ),
+           )
+    c = Config(
+        executors=[htex],
+        strategy='none',
+        strategy_period=0.5,
+    )
+    return c, htex
+
+
+@parsl.python_app
+def app():
+    return 7
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("msg",
+                         (b'FuzzyByte\rSTREAM',  # not JSON
+                          b'{}',  # missing fields
+                          b'{"type":"heartbeat"}',  # regression test #3262
+                          )
+                         )
+def test_bad_messages(try_assert, msg):
+    """This tests that the interchange is resilient to a few different bad
+    messages: malformed messages caused by implementation errors, and
+    heartbeat messages from managers that are not registered.
+
+    The heartbeat test is a regression test for issues #3262, #3632
+    """
+
+    c, htex = fresh_config()
+
+    with parsl.load(c):
+
+        # send a bad message into the interchange on the task_outgoing worker
+        # channel, and then check that the interchange is still alive enough
+        # that we can scale out a block and run a task.
+
+        (task_port, result_port) = htex.command_client.run("WORKER_PORTS")
+
+        context = zmq.Context()
+        channel_timeout = 10000  # in milliseconds
+        task_channel = context.socket(zmq.DEALER)
+        task_channel.setsockopt(zmq.LINGER, 0)
+        task_channel.setsockopt(zmq.IDENTITY, b'testid')
+
+        task_channel.set_hwm(0)
+        task_channel.setsockopt(zmq.SNDTIMEO, channel_timeout)
+        task_channel.connect(f"tcp://localhost:{task_port}")
+
+        task_channel.send(msg)
+
+        # If the interchange exits, it's likely that this test will hang rather
+        # than raise an error, because the interchange interaction code
+        # assumes the interchange is always there.
+        # In the case of issue #3262, an exception message goes to stderr, and
+        # no error goes to the interchange log file.
+        htex.scale_out_facade(1)
+        try_assert(lambda: len(htex.connected_managers()) == 1, timeout_ms=10000)
+
+        assert app().result() == 7

parsl/tests/test_serialization/test_3495_deserialize_managerlost.py

@@ -32,7 +32,7 @@ def test_manager_lost_system_failure(tmpd_cwd):
         cores_per_worker=1,
         worker_logdir_root=str(tmpd_cwd),
         heartbeat_period=1,
-        heartbeat_threshold=1,
+        heartbeat_threshold=3,
     )
     c = Config(executors=[hte], strategy='simple', strategy_period=0.1)
 

parsl/tests/test_staging/test_docs_1.py

@@ -12,6 +12,7 @@ def convert(inputs=[], outputs=[]):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test():
     # create an remote Parsl file
     inp = File('ftp://ftp.iana.org/pub/mirror/rirstats/arin/ARIN-STATS-FORMAT-CHANGE.txt')

parsl/tests/test_staging/test_output_chain_filenames.py

@@ -1,5 +1,7 @@
 from concurrent.futures import Future
 
+import pytest
+
 from parsl import File
 from parsl.app.app import bash_app
 
@@ -14,6 +16,7 @@ def app2(inputs=(), outputs=(), stdout=None, stderr=None, mock=False):
     return f"echo '{inputs[0]}' > {outputs[0]}"
 
 
+@pytest.mark.shared_fs
 def test_behavior(tmpd_cwd):
     expected_path = str(tmpd_cwd / "simple-out.txt")
     app1_future = app1(

parsl/tests/test_staging/test_staging_ftp.py

@@ -15,6 +15,7 @@ def sort_strings(inputs=[], outputs=[]):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_ftp():
     """Test staging for an ftp file
 

parsl/tests/test_staging/test_staging_https.py

@@ -48,6 +48,7 @@ def sort_strings_additional_executor(inputs=(), outputs=()):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_https_cleannet(tmpd_cwd):
     unsorted_file = File(_unsorted_url)
     sorted_file = File(tmpd_cwd / 'sorted.txt')
@@ -68,6 +69,7 @@ def test_staging_https_local(tmpd_cwd):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_https_kwargs(tmpd_cwd):
     unsorted_file = File(_unsorted_url)
     sorted_file = File(tmpd_cwd / 'sorted.txt')
@@ -78,6 +80,7 @@ def test_staging_https_kwargs(tmpd_cwd):
 
 
 @pytest.mark.cleannet
+@pytest.mark.staging_required
 def test_staging_https_args(tmpd_cwd):
     unsorted_file = File(_unsorted_url)
     sorted_file = File(tmpd_cwd / 'sorted.txt')

parsl/tests/test_staging/test_staging_stdout.py

@@ -15,6 +15,7 @@ def output_to_stds(*, stdout=parsl.AUTO_LOGNAME, stderr=parsl.AUTO_LOGNAME):
     return "echo hello ; echo goodbye >&2"
 
 
+@pytest.mark.staging_required
 def test_stdout_staging_file(tmpd_cwd, caplog):
     basename = str(tmpd_cwd) + "/stdout.txt"
     stdout_file = File("file://" + basename)
@@ -30,6 +31,7 @@ def test_stdout_staging_file(tmpd_cwd, caplog):
         assert record.levelno < logging.ERROR
 
 
+@pytest.mark.staging_required
 def test_stdout_stderr_staging_zip(tmpd_cwd, caplog):
     zipfile_name = str(tmpd_cwd) + "/staging.zip"
     stdout_relative_path = "somewhere/test-out.txt"

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.10.21'
+VERSION = '2024.11.04'

{parsl-2024.10.21.data → parsl-2024.11.4.data}/scripts/interchange.py

@@ -66,7 +66,7 @@ class Interchange:
              If specified the interchange will only listen on this address for connections from workers
              else, it binds to all addresses.
 
-        client_ports : triple(int, int, int)
+        client_ports : tuple(int, int, int)
              The ports at which the client can be reached
 
         worker_ports : tuple(int, int)
@@ -104,7 +104,6 @@ class Interchange:
         os.makedirs(self.logdir, exist_ok=True)
 
         start_file_logger("{}/interchange.log".format(self.logdir), level=logging_level)
-        logger.propagate = False
         logger.debug("Initializing Interchange process")
 
         self.client_address = client_address
@@ -437,9 +436,13 @@ class Interchange:
                     logger.info(f"Manager {manager_id!r} has compatible Parsl version {msg['parsl_v']}")
                     logger.info(f"Manager {manager_id!r} has compatible Python version {msg['python_v'].rsplit('.', 1)[0]}")
             elif msg['type'] == 'heartbeat':
-                self._ready_managers[manager_id]['last_heartbeat'] = time.time()
-                logger.debug("Manager %r sent heartbeat via tasks connection", manager_id)
-                self.task_outgoing.send_multipart([manager_id, b'', PKL_HEARTBEAT_CODE])
+                manager = self._ready_managers.get(manager_id)
+                if manager:
+                    manager['last_heartbeat'] = time.time()
+                    logger.debug("Manager %r sent heartbeat via tasks connection", manager_id)
+                    self.task_outgoing.send_multipart([manager_id, b'', PKL_HEARTBEAT_CODE])
+                else:
+                    logger.warning("Received heartbeat via tasks connection for not-registered manager %r", manager_id)
             elif msg['type'] == 'drain':
                 self._ready_managers[manager_id]['draining'] = True
                 logger.debug("Manager %r requested drain", manager_id)

{parsl-2024.10.21.data → parsl-2024.11.4.data}/scripts/process_worker_pool.py

@@ -650,14 +650,6 @@ def worker(
     debug: bool,
     mpi_launcher: str,
 ):
-    """
-
-    Put request token into queue
-    Get task from task_queue
-    Pop request from queue
-    Put result into result_queue
-    """
-
     # override the global logger inherited from the __main__ process (which
     # usually logs to manager.log) with one specific to this worker.
     global logger

{parsl-2024.10.21.dist-info → parsl-2024.11.4.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.10.21
+Version: 2024.11.4
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.10.21.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.11.04.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0