parsl 2024.3.18__py3-none-any.whl → 2024.4.1__py3-none-any.whl

parsl/monitoring/router.py

@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+import os
+import socket
+import time
+import pickle
+import logging
+import zmq
+
+import queue
+
+from parsl.log_utils import set_file_logger
+from parsl.process_loggers import wrap_with_logs
+from parsl.utils import setproctitle
+
+from parsl.monitoring.message_type import MessageType
+from parsl.monitoring.types import AddressedMonitoringMessage, TaggedMonitoringMessage
+from typing import Optional, Tuple, Union
+
+
+logger = logging.getLogger(__name__)
+
+
+class MonitoringRouter:
+
+    def __init__(self,
+                 *,
+                 hub_address: str,
+                 udp_port: Optional[int] = None,
+                 zmq_port_range: Tuple[int, int] = (55050, 56000),
+
+                 monitoring_hub_address: str = "127.0.0.1",
+                 logdir: str = ".",
+                 run_id: str,
+                 logging_level: int = logging.INFO,
+                 atexit_timeout: int = 3    # in seconds
+                 ):
+        """ Initializes a monitoring configuration class.
+
+        Parameters
+        ----------
+        hub_address : str
+             The ip address at which the workers will be able to reach the Hub.
+        udp_port : int
+             The specific port at which workers will be able to reach the Hub via UDP. Default: None
+        zmq_port_range : tuple(int, int)
+             The MonitoringHub picks ports at random from the range which will be used by Hub.
+             Default: (55050, 56000)
+        logdir : str
+             Parsl log directory paths. Logs and temp files go here. Default: '.'
+        logging_level : int
+             Logging level as defined in the logging module. Default: logging.INFO
+        atexit_timeout : float, optional
+            The amount of time in seconds to terminate the hub without receiving any messages, after the last dfk workflow message is received.
+
+        """
+        os.makedirs(logdir, exist_ok=True)
+        self.logger = set_file_logger("{}/monitoring_router.log".format(logdir),
+                                      name="monitoring_router",
+                                      level=logging_level)
+        self.logger.debug("Monitoring router starting")
+
+        self.hub_address = hub_address
+        self.atexit_timeout = atexit_timeout
+        self.run_id = run_id
+
+        self.loop_freq = 10.0  # milliseconds
+
+        # Initialize the UDP socket
+        self.udp_sock = socket.socket(socket.AF_INET,
+                                      socket.SOCK_DGRAM,
+                                      socket.IPPROTO_UDP)
+
+        # We are trying to bind to all interfaces with 0.0.0.0
+        if not udp_port:
+            self.udp_sock.bind(('0.0.0.0', 0))
+            self.udp_port = self.udp_sock.getsockname()[1]
+        else:
+            self.udp_port = udp_port
+            try:
+                self.udp_sock.bind(('0.0.0.0', self.udp_port))
+            except Exception as e:
+                raise RuntimeError(f"Could not bind to udp_port {udp_port} because: {e}")
+        self.udp_sock.settimeout(self.loop_freq / 1000)
+        self.logger.info("Initialized the UDP socket on 0.0.0.0:{}".format(self.udp_port))
+
+        self._context = zmq.Context()
+        self.zmq_receiver_channel = self._context.socket(zmq.DEALER)
+        self.zmq_receiver_channel.setsockopt(zmq.LINGER, 0)
+        self.zmq_receiver_channel.set_hwm(0)
+        self.zmq_receiver_channel.RCVTIMEO = int(self.loop_freq)  # in milliseconds
+        self.logger.debug("hub_address: {}. zmq_port_range {}".format(hub_address, zmq_port_range))
+        self.zmq_receiver_port = self.zmq_receiver_channel.bind_to_random_port("tcp://*",
+                                                                               min_port=zmq_port_range[0],
+                                                                               max_port=zmq_port_range[1])
+
+    def start(self,
+              priority_msgs: "queue.Queue[AddressedMonitoringMessage]",
+              node_msgs: "queue.Queue[AddressedMonitoringMessage]",
+              block_msgs: "queue.Queue[AddressedMonitoringMessage]",
+              resource_msgs: "queue.Queue[AddressedMonitoringMessage]") -> None:
+        try:
+            router_keep_going = True
+            while router_keep_going:
+                try:
+                    data, addr = self.udp_sock.recvfrom(2048)
+                    resource_msg = pickle.loads(data)
+                    self.logger.debug("Got UDP Message from {}: {}".format(addr, resource_msg))
+                    resource_msgs.put((resource_msg, addr))
+                except socket.timeout:
+                    pass
+
+                try:
+                    dfk_loop_start = time.time()
+                    while time.time() - dfk_loop_start < 1.0:  # TODO make configurable
+                        # note that nothing checks that msg really is of the annotated type
+                        msg: TaggedMonitoringMessage
+                        msg = self.zmq_receiver_channel.recv_pyobj()
+
+                        assert isinstance(msg, tuple), "ZMQ Receiver expects only tuples, got {}".format(msg)
+                        assert len(msg) >= 1, "ZMQ Receiver expects tuples of length at least 1, got {}".format(msg)
+                        assert len(msg) == 2, "ZMQ Receiver expects message tuples of exactly length 2, got {}".format(msg)
+
+                        msg_0: AddressedMonitoringMessage
+                        msg_0 = (msg, 0)
+
+                        if msg[0] == MessageType.NODE_INFO:
+                            msg[1]['run_id'] = self.run_id
+                            node_msgs.put(msg_0)
+                        elif msg[0] == MessageType.RESOURCE_INFO:
+                            resource_msgs.put(msg_0)
+                        elif msg[0] == MessageType.BLOCK_INFO:
+                            block_msgs.put(msg_0)
+                        elif msg[0] == MessageType.TASK_INFO:
+                            priority_msgs.put(msg_0)
+                        elif msg[0] == MessageType.WORKFLOW_INFO:
+                            priority_msgs.put(msg_0)
+                            if 'exit_now' in msg[1] and msg[1]['exit_now']:
+                                router_keep_going = False
+                        else:
+                            # There is a type: ignore here because if msg[0]
+                            # is of the correct type, this code is unreachable,
+                            # but there is no verification that the message
+                            # received from zmq_receiver_channel.recv_pyobj() is actually
+                            # of that type.
+                            self.logger.error("Discarding message "  # type: ignore[unreachable]
+                                              f"from interchange with unknown type {msg[0].value}")
+                except zmq.Again:
+                    pass
+                except Exception:
+                    # This will catch malformed messages. What happens if the
+                    # channel is broken in such a way that it always raises
+                    # an exception? Looping on this would maybe be the wrong
+                    # thing to do.
+                    self.logger.warning("Failure processing a ZMQ message", exc_info=True)
+
+            self.logger.info("Monitoring router draining")
+            last_msg_received_time = time.time()
+            while time.time() - last_msg_received_time < self.atexit_timeout:
+                try:
+                    data, addr = self.udp_sock.recvfrom(2048)
+                    msg = pickle.loads(data)
+                    self.logger.debug("Got UDP Message from {}: {}".format(addr, msg))
+                    resource_msgs.put((msg, addr))
+                    last_msg_received_time = time.time()
+                except socket.timeout:
+                    pass
+
+            self.logger.info("Monitoring router finishing normally")
+        finally:
+            self.logger.info("Monitoring router finished")
+
+
+@wrap_with_logs
+def router_starter(comm_q: "queue.Queue[Union[Tuple[int, int], str]]",
+                   exception_q: "queue.Queue[Tuple[str, str]]",
+                   priority_msgs: "queue.Queue[AddressedMonitoringMessage]",
+                   node_msgs: "queue.Queue[AddressedMonitoringMessage]",
+                   block_msgs: "queue.Queue[AddressedMonitoringMessage]",
+                   resource_msgs: "queue.Queue[AddressedMonitoringMessage]",
+
+                   hub_address: str,
+                   udp_port: Optional[int],
+                   zmq_port_range: Tuple[int, int],
+
+                   logdir: str,
+                   logging_level: int,
+                   run_id: str) -> None:
+    setproctitle("parsl: monitoring router")
+    try:
+        router = MonitoringRouter(hub_address=hub_address,
+                                  udp_port=udp_port,
+                                  zmq_port_range=zmq_port_range,
+                                  logdir=logdir,
+                                  logging_level=logging_level,
+                                  run_id=run_id)
+    except Exception as e:
+        logger.error("MonitoringRouter construction failed.", exc_info=True)
+        comm_q.put(f"Monitoring router construction failed: {e}")
+    else:
+        comm_q.put((router.udp_port, router.zmq_receiver_port))
+
+        router.logger.info("Starting MonitoringRouter in router_starter")
+        try:
+            router.start(priority_msgs, node_msgs, block_msgs, resource_msgs)
+        except Exception as e:
+            router.logger.exception("router.start exception")
+            exception_q.put(('Hub', str(e)))

parsl/tests/site_tests/test_provider.py

@@ -58,7 +58,7 @@ def test_provider():
     logger.info("Job in terminal state")
 
     _, current_jobs = executor._get_block_and_job_ids()
-    # PR 1952 stoped removing scale_in blocks from self.blocks
+    # PR 1952 stoped removing scale_in blocks from self.blocks_to_job_id
     # A new PR will handle removing blocks from self.block
     # this includes failed/completed/canceled blocks
     assert len(current_jobs) == 1, "Expected current_jobs == 1"

parsl/tests/test_htex/test_disconnected_blocks.py

@@ -26,7 +26,6 @@ def local_config():
                 ),
             )
         ],
-        run_dir="/tmp/test_htex",
         max_idletime=0.5,
         strategy='htex_auto_scale',
     )

parsl/tests/test_htex/test_drain.py

@@ -35,6 +35,7 @@ def local_config():
             )
         ],
         strategy='none',
+        strategy_period=0.1
     )
 
 

parsl/tests/test_monitoring/test_fuzz_zmq.py

@@ -41,11 +41,11 @@ def test_row_counts():
 
     # dig out the interchange port...
     hub_address = parsl.dfk().hub_address
-    hub_interchange_port = parsl.dfk().hub_interchange_port
+    hub_zmq_port = parsl.dfk().hub_zmq_port
 
     # this will send a string to a new socket connection
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.connect((hub_address, hub_interchange_port))
+        s.connect((hub_address, hub_zmq_port))
         s.sendall(b'fuzzing\r')
 
     # this will send a non-object down the DFK's existing ZMQ connection

parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py

@@ -0,0 +1,85 @@
+import logging
+import os
+import parsl
+import pytest
+import time
+
+from parsl.providers import LocalProvider
+from parsl.channels import LocalChannel
+from parsl.launchers import SimpleLauncher
+
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.monitoring import MonitoringHub
+
+
+def fresh_config(run_dir, strategy, db_url):
+    return Config(
+        run_dir=os.fspath(run_dir),
+        executors=[
+            HighThroughputExecutor(
+                label="htex_local",
+                cores_per_worker=1,
+                encrypted=True,
+                provider=LocalProvider(
+                    channel=LocalChannel(),
+                    init_blocks=1,
+                    # min and max are set to 0 to ensure that we don't get
+                    # a block from ongoing strategy scaling, only from
+                    # init_blocks
+                    min_blocks=0,
+                    max_blocks=0,
+                    launcher=SimpleLauncher(),
+                ),
+            )
+        ],
+        strategy=strategy,
+        strategy_period=0.1,
+        monitoring=MonitoringHub(
+                        hub_address="localhost",
+                        hub_port=55055,
+                        logging_endpoint=db_url
+        )
+    )
+
+
+@parsl.python_app
+def this_app():
+    pass
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("strategy", ('none', 'simple', 'htex_auto_scale'))
+def test_row_counts(tmpd_cwd, strategy):
+    # this is imported here rather than at module level because
+    # it isn't available in a plain parsl install, so this module
+    # would otherwise fail to import and break even a basic test
+    # run.
+    import sqlalchemy
+    from sqlalchemy import text
+
+    db_url = f"sqlite:///{tmpd_cwd}/monitoring.db"
+    with parsl.load(fresh_config(tmpd_cwd, strategy, db_url)):
+        dfk = parsl.dfk()
+        run_id = dfk.run_id
+
+        this_app().result()
+
+    parsl.clear()
+
+    engine = sqlalchemy.create_engine(db_url)
+    with engine.begin() as connection:
+
+        binds = {"run_id": run_id}
+
+        result = connection.execute(text("SELECT COUNT(DISTINCT block_id) FROM block WHERE run_id = :run_id"), binds)
+        (c, ) = result.first()
+        assert c == 1, "We should see a single block in this database"
+
+        result = connection.execute(text("SELECT COUNT(*) FROM block WHERE block_id = 0 AND status = 'PENDING' AND run_id = :run_id"), binds)
+        (c, ) = result.first()
+        assert c == 1, "There should be a single pending status"
+
+        result = connection.execute(text("SELECT COUNT(*) FROM block WHERE block_id = 0 AND status = 'CANCELLED' AND run_id = :run_id"), binds)
+        (c, ) = result.first()
+        assert c == 1, "There should be a single cancelled status"

parsl/tests/test_python_apps/test_context_manager.py

@@ -0,0 +1,40 @@
+import parsl
+from parsl.tests.configs.local_threads import fresh_config
+import pytest
+from parsl.errors import NoDataFlowKernelError
+from parsl.dataflow.dflow import DataFlowKernel
+
+
+@parsl.python_app
+def square(x):
+    return x * x
+
+
+@parsl.bash_app
+def foo(x, stdout='foo.stdout'):
+    return f"echo {x + 1}"
+
+
+def local_setup():
+    pass
+
+
+def local_teardown():
+    parsl.clear()
+
+
+@pytest.mark.local
+def test_within_context_manger():
+    config = fresh_config()
+    with parsl.load(config=config) as dfk:
+        assert isinstance(dfk, DataFlowKernel)
+
+        bash_future = foo(1)
+        assert bash_future.result() == 0
+
+        with open('foo.stdout', 'r') as f:
+            assert f.read() == "2\n"
+
+    with pytest.raises(NoDataFlowKernelError) as excinfo:
+        square(2).result()
+    assert str(excinfo.value) == "Cannot submit to a DFK that has been cleaned up"

parsl/tests/test_scaling/test_shutdown_scalein.py

@@ -0,0 +1,78 @@
+import threading
+
+import pytest
+
+import parsl
+from parsl.channels import LocalChannel
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
+
+import random
+
+# we need some blocks, but it doesn't matter too much how many, as long
+# as they can all start up and get registered within the try_assert
+# timeout later on.
+BLOCK_COUNT = 3
+
+
+class AccumulatingLocalProvider(LocalProvider):
+    def __init__(self, *args, **kwargs):
+        # Use a list for submitted job IDs because if there are multiple
+        # submissions returning the same job ID, this test should count
+        # those...
+        self.submit_job_ids = []
+
+        # ... but there's no requirement, I think, that cancel must be called
+        # only once per job id. What matters here is that each job ID is
+        # cancelled at least once.
+        self.cancel_job_ids = set()
+
+        super().__init__(*args, **kwargs)
+
+    def submit(self, *args, **kwargs):
+        job_id = super().submit(*args, **kwargs)
+        self.submit_job_ids.append(job_id)
+        return job_id
+
+    def cancel(self, job_ids):
+        self.cancel_job_ids.update(job_ids)
+        return super().cancel(job_ids)
+
+
+@pytest.mark.local
+def test_shutdown_scalein_blocks(tmpd_cwd, try_assert):
+    """
+    This test scales up several blocks, and then checks that they are all
+    scaled in at DFK shutdown.
+    """
+    accumulating_provider = AccumulatingLocalProvider(
+        channel=LocalChannel(),
+        init_blocks=BLOCK_COUNT,
+        min_blocks=0,
+        max_blocks=0,
+        launcher=SimpleLauncher(),
+    )
+
+    htex = HighThroughputExecutor(
+               label="htex_local",
+               cores_per_worker=1,
+               provider=accumulating_provider
+           )
+
+    config = Config(
+        executors=[htex],
+        strategy='none',
+        strategy_period=0.1,
+        run_dir=str(tmpd_cwd)
+    )
+
+    with parsl.load(config):
+        # this will wait for everything to be scaled out fully
+        try_assert(lambda: len(htex.connected_managers()) == BLOCK_COUNT)
+
+    parsl.clear()
+
+    assert len(accumulating_provider.submit_job_ids) == BLOCK_COUNT, f"Exactly {BLOCK_COUNT} blocks should have been launched"
+    assert len(accumulating_provider.cancel_job_ids) == BLOCK_COUNT, f"Exactly {BLOCK_COUNT} blocks should have been scaled in"

parsl/tests/test_shutdown/test_kill_monitoring.py

@@ -0,0 +1,65 @@
+import os
+import parsl
+import pytest
+import signal
+import time
+
+from parsl.tests.configs.htex_local_alternate import fresh_config
+
+# This is a very generous upper bound on expected shutdown time of target
+# process after receiving a signal, measured in seconds.
+PERMITTED_SHUTDOWN_TIME_S = 60
+
+
+@parsl.python_app
+def simple_app():
+    return True
+
+
+@pytest.mark.local
+def test_no_kills():
+    """This tests that we can create a monitoring-enabled DFK and shut it down."""
+
+    parsl.load(fresh_config())
+
+    assert parsl.dfk().monitoring is not None, "This test requires monitoring"
+
+    parsl.dfk().cleanup()
+    parsl.clear()
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("sig", [signal.SIGINT, signal.SIGTERM, signal.SIGKILL, signal.SIGQUIT])
+@pytest.mark.parametrize("process_attr", ["router_proc", "dbm_proc"])
+def test_kill_monitoring_helper_process(sig, process_attr, try_assert):
+    """This tests that we can kill a monitoring process and still have successful shutdown.
+    SIGINT emulates some racy behaviour when ctrl-C is pressed: that
+    monitoring processes receive a ctrl-C too, and so the other processes
+    need to be tolerant to monitoring processes arbitrarily exiting.
+    """
+
+    parsl.load(fresh_config())
+
+    dfk = parsl.dfk()
+
+    assert dfk.monitoring is not None, "Monitoring required"
+
+    target_proc = getattr(dfk.monitoring, process_attr)
+
+    assert target_proc is not None, "prereq: target process must exist"
+    assert target_proc.is_alive(), "prereq: target process must be alive"
+
+    target_pid = target_proc.pid
+    assert target_pid is not None, "prereq: target process must have a pid"
+
+    os.kill(target_pid, sig)
+
+    try_assert(lambda: not target_proc.is_alive(), timeout_ms=PERMITTED_SHUTDOWN_TIME_S * 1000)
+
+    # now we have broken one piece of the monitoring system, do some app
+    # execution and then shut down.
+
+    simple_app().result()
+
+    parsl.dfk().cleanup()
+    parsl.clear()

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.03.18'
+VERSION = '2024.04.01'

{parsl-2024.3.18.data → parsl-2024.4.1.data}/scripts/process_worker_pool.py

@@ -335,14 +335,17 @@ class Manager:
                 self.heartbeat_to_incoming()
                 last_beat = time.time()
 
-            if self.drain_time and time.time() > self.drain_time:
+            if time.time() > self.drain_time:
                 logger.info("Requesting drain")
                 self.drain_to_incoming()
-                self.drain_time = None
                 # This will start the pool draining...
                 # Drained exit behaviour does not happen here. It will be
                 # driven by the interchange sending a DRAINED_CODE message.
 
+                # now set drain time to the far future so we don't send a drain
+                # message every iteration.
+                self.drain_time = float('inf')
+
             poll_duration_s = max(0, next_interesting_event_time - time.time())
             socks = dict(poller.poll(timeout=poll_duration_s * 1000))
 

{parsl-2024.3.18.dist-info → parsl-2024.4.1.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.3.18
+Version: 2024.4.1
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.03.18.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.04.01.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -54,7 +54,7 @@ Requires-Dist: pyyaml ; extra == 'all'
 Requires-Dist: cffi ; extra == 'all'
 Requires-Dist: jsonschema ; extra == 'all'
 Requires-Dist: proxystore ; extra == 'all'
-Requires-Dist: radical.pilot ; extra == 'all'
+Requires-Dist: radical.pilot ==1.47 ; extra == 'all'
 Provides-Extra: aws
 Requires-Dist: boto3 ; extra == 'aws'
 Provides-Extra: azure
@@ -83,7 +83,7 @@ Requires-Dist: oauth-ssh >=0.9 ; extra == 'oauth_ssh'
 Provides-Extra: proxystore
 Requires-Dist: proxystore ; extra == 'proxystore'
 Provides-Extra: radical-pilot
-Requires-Dist: radical.pilot ; extra == 'radical-pilot'
+Requires-Dist: radical.pilot ==1.47 ; extra == 'radical-pilot'
 Provides-Extra: visualization
 Requires-Dist: pydot ; extra == 'visualization'
 Requires-Dist: networkx <2.6,>=2.5 ; extra == 'visualization'