parsl 2024.3.18__py3-none-any.whl → 2024.3.25__py3-none-any.whl

parsl/dataflow/dflow.py

@@ -108,12 +108,12 @@ class DataFlowKernel:
 
         # hub address and port for interchange to connect
         self.hub_address = None  # type: Optional[str]
-        self.hub_interchange_port = None  # type: Optional[int]
+        self.hub_zmq_port = None  # type: Optional[int]
         if self.monitoring:
             if self.monitoring.logdir is None:
                 self.monitoring.logdir = self.run_dir
             self.hub_address = self.monitoring.hub_address
-            self.hub_interchange_port = self.monitoring.start(self.run_id, self.run_dir, self.config.run_dir)
+            self.hub_zmq_port = self.monitoring.start(self.run_id, self.run_dir, self.config.run_dir)
 
         self.time_began = datetime.datetime.now()
         self.time_completed: Optional[datetime.datetime] = None
@@ -206,6 +206,13 @@ class DataFlowKernel:
 
         atexit.register(self.atexit_cleanup)
 
+    def __enter__(self):
+        pass
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        logger.debug("Exiting the context manager, calling cleanup for DFK")
+        self.cleanup()
+
     def _send_task_log_info(self, task_record: TaskRecord) -> None:
         if self.monitoring:
             task_log_info = self._create_task_log_info(task_record)
@@ -1115,12 +1122,12 @@ class DataFlowKernel:
 
         channel.makedirs(channel.script_dir, exist_ok=True)
 
-    def add_executors(self, executors):
+    def add_executors(self, executors: Sequence[ParslExecutor]) -> None:
         for executor in executors:
             executor.run_id = self.run_id
             executor.run_dir = self.run_dir
             executor.hub_address = self.hub_address
-            executor.hub_port = self.hub_interchange_port
+            executor.hub_port = self.hub_zmq_port
             if hasattr(executor, 'provider'):
                 if hasattr(executor.provider, 'script_dir'):
                     executor.provider.script_dir = os.path.join(self.run_dir, 'submit_scripts')

parsl/executors/base.py

@@ -106,6 +106,16 @@ class ParslExecutor(metaclass=ABCMeta):
     def run_dir(self, value: str) -> None:
         self._run_dir = value
 
+    @property
+    def run_id(self) -> Optional[str]:
+        """UUID for the enclosing DFK.
+        """
+        return self._run_id
+
+    @run_id.setter
+    def run_id(self, value: Optional[str]) -> None:
+        self._run_id = value
+
     @property
     def hub_address(self) -> Optional[str]:
         """Address to the Hub for monitoring.

parsl/executors/status_handling.py

@@ -61,7 +61,7 @@ class BlockProviderExecutor(ParslExecutor):
         # errors can happen during the submit call to the provider; this is used
         # to keep track of such errors so that they can be handled in one place
         # together with errors reported by status()
-        self._simulated_status: Dict[Any, JobStatus] = {}
+        self._simulated_status: Dict[str, JobStatus] = {}
         self._executor_bad_state = threading.Event()
         self._executor_exception: Optional[Exception] = None
 
@@ -102,13 +102,10 @@ class BlockProviderExecutor(ParslExecutor):
         else:
             return self._provider.status_polling_interval
 
-    def _fail_job_async(self, block_id: Any, message: str):
+    def _fail_job_async(self, block_id: str, message: str):
         """Marks a job that has failed to start but would not otherwise be included in status()
         as failed and report it in status()
         """
-        if block_id is None:
-            block_id = str(self._block_id_counter.get_id())
-            logger.info(f"Allocated block ID {block_id} for simulated failure")
         self._simulated_status[block_id] = JobStatus(JobState.FAILED, message)
 
     @abstractproperty
@@ -211,10 +208,6 @@ class BlockProviderExecutor(ParslExecutor):
 
         Cause the executor to reduce the number of blocks by count.
 
-        We should have the scale in method simply take resource object
-        which will have the scaling methods, scale_in itself should be a coroutine, since
-        scaling tasks can be slow.
-
         :return: A list of block ids corresponding to the blocks that were removed.
         """
         pass

parsl/executors/taskvine/executor.py

@@ -4,6 +4,7 @@ high-throughput system for delegating Parsl tasks to thousands of remote machine
 """
 
 # Import Python built-in libraries
+import atexit
 import threading
 import multiprocessing
 import logging
@@ -171,7 +172,7 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         # Path to directory that holds all tasks' data and results.
         self._function_data_dir = ""
 
-        # helper scripts to prepare package tarballs for Parsl apps
+        # Helper scripts to prepare package tarballs for Parsl apps
         self._package_analyze_script = shutil.which("poncho_package_analyze")
         self._package_create_script = shutil.which("poncho_package_create")
         if self._package_analyze_script is None or self._package_create_script is None:
@@ -179,6 +180,18 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         else:
             self._poncho_available = True
 
+        # Register atexit handler to cleanup when Python shuts down
+        atexit.register(self.atexit_cleanup)
+
+        # Attribute indicating whether this executor was started to shut it down properly.
+        # This safeguards cases where an object of this executor is created but
+        # the executor never starts, so it shouldn't be shutdowned.
+        self._started = False
+
+    def atexit_cleanup(self):
+        # Calls this executor's shutdown method upon Python exiting the process.
+        self.shutdown()
+
     def _get_launch_command(self, block_id):
         # Implements BlockProviderExecutor's abstract method.
         # This executor uses different terminology for worker/launch
@@ -238,6 +251,9 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         retrieve Parsl tasks within the TaskVine system.
         """
 
+        # Mark this executor object as started
+        self._started = True
+
         # Synchronize connection and communication settings between the manager and factory
         self.__synchronize_manager_factory_comm_settings()
 
@@ -598,6 +614,10 @@ class TaskVineExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         """Shutdown the executor. Sets flag to cancel the submit process and
         collector thread, which shuts down the TaskVine system submission.
         """
+        if not self._started:
+            # Don't shutdown if the executor never starts.
+            return
+
         logger.debug("TaskVine shutdown started")
         self._should_stop.set()
 

parsl/executors/workqueue/executor.py

@@ -3,6 +3,7 @@ Cooperative Computing Lab (CCL) at Notre Dame to provide a fault-tolerant,
 high-throughput system for delegating Parsl tasks to thousands of remote machines
 """
 
+import atexit
 import threading
 import multiprocessing
 import logging
@@ -298,6 +299,18 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         if self.init_command != "":
             self.launch_cmd = self.init_command + "; " + self.launch_cmd
 
+        # register atexit handler to cleanup when Python shuts down
+        atexit.register(self.atexit_cleanup)
+
+        # Attribute indicating whether this executor was started to shut it down properly.
+        # This safeguards cases where an object of this executor is created but
+        # the executor never starts, so it shouldn't be shutdowned.
+        self.started = False
+
+    def atexit_cleanup(self):
+        # Calls this executor's shutdown method upon Python exiting the process.
+        self.shutdown()
+
     def _get_launch_command(self, block_id):
         # this executor uses different terminology for worker/launch
         # commands than in htex
@@ -307,6 +320,8 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         """Create submit process and collector thread to create, send, and
         retrieve Parsl tasks within the Work Queue system.
         """
+        # Mark this executor object as started
+        self.started = True
         self.tasks_lock = threading.Lock()
 
         # Create directories for data and results
@@ -695,6 +710,10 @@ class WorkQueueExecutor(BlockProviderExecutor, putils.RepresentationMixin):
         """Shutdown the executor. Sets flag to cancel the submit process and
         collector thread, which shuts down the Work Queue system submission.
         """
+        if not self.started:
+            # Don't shutdown if the executor never starts.
+            return
+
         logger.debug("Work Queue shutdown started")
         self.should_stop.value = True
 

parsl/jobs/error_handlers.py

@@ -20,7 +20,7 @@ def simple_error_handler(executor: status_handling.BlockProviderExecutor, status
         executor.set_bad_state_and_fail_all(_get_error(status))
 
 
-def windowed_error_handler(executor: status_handling.BlockProviderExecutor, status: Dict[str, JobStatus], threshold: int = 3):
+def windowed_error_handler(executor: status_handling.BlockProviderExecutor, status: Dict[str, JobStatus], threshold: int = 3) -> None:
     sorted_status = [(key, status[key]) for key in sorted(status, key=lambda x: int(x))]
     current_window = dict(sorted_status[-threshold:])
     total, failed = _count_jobs(current_window)

parsl/jobs/job_status_poller.py

@@ -29,7 +29,7 @@ class PollItem:
         if self._dfk and self._dfk.monitoring is not None:
             self.monitoring_enabled = True
             hub_address = self._dfk.hub_address
-            hub_port = self._dfk.hub_interchange_port
+            hub_port = self._dfk.hub_zmq_port
             context = zmq.Context()
             self.hub_channel = context.socket(zmq.DEALER)
             self.hub_channel.set_hwm(0)
@@ -72,7 +72,7 @@ class PollItem:
     def executor(self) -> BlockProviderExecutor:
         return self._executor
 
-    def scale_in(self, n, max_idletime=None):
+    def scale_in(self, n: int, max_idletime: Optional[float] = None) -> List[str]:
 
         if max_idletime is None:
             block_ids = self._executor.scale_in(n)
@@ -82,7 +82,7 @@ class PollItem:
             # scale_in method really does come from HighThroughputExecutor,
             # and so does have an extra max_idletime parameter not present
             # in the executor interface.
-            block_ids = self._executor.scale_in(n, max_idletime=max_idletime)
+            block_ids = self._executor.scale_in(n, max_idletime=max_idletime)  # type: ignore[call-arg]
         if block_ids is not None:
             new_status = {}
             for block_id in block_ids:
@@ -91,7 +91,7 @@ class PollItem:
             self.send_monitoring_info(new_status)
         return block_ids
 
-    def scale_out(self, n):
+    def scale_out(self, n: int) -> List[str]:
         block_ids = self._executor.scale_out(n)
         if block_ids is not None:
             new_status = {}

parsl/monitoring/monitoring.py

@@ -1,9 +1,7 @@
 from __future__ import annotations
 
 import os
-import socket
 import time
-import pickle
 import logging
 import typeguard
 import zmq
@@ -22,8 +20,9 @@ from parsl.utils import setproctitle
 
 from parsl.serialize import deserialize
 
+from parsl.monitoring.router import router_starter
 from parsl.monitoring.message_type import MessageType
-from parsl.monitoring.types import AddressedMonitoringMessage, TaggedMonitoringMessage
+from parsl.monitoring.types import AddressedMonitoringMessage
 from typing import cast, Any, Callable, Dict, Optional, Sequence, Tuple, Union, TYPE_CHECKING
 
 _db_manager_excepts: Optional[Exception]
@@ -93,8 +92,6 @@ class MonitoringHub(RepresentationMixin):
              Default: 30 seconds
         """
 
-        self.logger = logger
-
         # Any is used to disable typechecking on uses of _dfk_channel,
         # because it is used in the code as if it points to a channel, but
         # the static type is that it can also be None. The code relies on
@@ -120,6 +117,8 @@ class MonitoringHub(RepresentationMixin):
 
     def start(self, run_id: str, dfk_run_dir: str, config_run_dir: Union[str, os.PathLike]) -> int:
 
+        logger.debug("Starting MonitoringHub")
+
         if self.logdir is None:
             self.logdir = "."
 
@@ -128,9 +127,6 @@ class MonitoringHub(RepresentationMixin):
 
         os.makedirs(self.logdir, exist_ok=True)
 
-        # Initialize the ZMQ pipe to the Parsl Client
-
-        self.logger.debug("Initializing ZMQ Pipes to client")
         self.monitoring_hub_active = True
 
         # This annotation is incompatible with typeguard 4.x instrumentation
@@ -166,8 +162,8 @@ class MonitoringHub(RepresentationMixin):
         self.router_proc = ForkProcess(target=router_starter,
                                        args=(comm_q, self.exception_q, self.priority_msgs, self.node_msgs, self.block_msgs, self.resource_msgs),
                                        kwargs={"hub_address": self.hub_address,
-                                               "hub_port": self.hub_port,
-                                               "hub_port_range": self.hub_port_range,
+                                               "udp_port": self.hub_port,
+                                               "zmq_port_range": self.hub_port_range,
                                                "logdir": self.logdir,
                                                "logging_level": logging.DEBUG if self.monitoring_debug else logging.INFO,
                                                "run_id": run_id
@@ -187,7 +183,7 @@ class MonitoringHub(RepresentationMixin):
                                     daemon=True,
                                     )
         self.dbm_proc.start()
-        self.logger.info("Started the router process {} and DBM process {}".format(self.router_proc.pid, self.dbm_proc.pid))
+        logger.info("Started the router process {} and DBM process {}".format(self.router_proc.pid, self.dbm_proc.pid))
 
         self.filesystem_proc = Process(target=filesystem_receiver,
                                        args=(self.logdir, self.resource_msgs, dfk_run_dir),
@@ -195,19 +191,19 @@ class MonitoringHub(RepresentationMixin):
                                        daemon=True
                                        )
         self.filesystem_proc.start()
-        self.logger.info(f"Started filesystem radio receiver process {self.filesystem_proc.pid}")
+        logger.info(f"Started filesystem radio receiver process {self.filesystem_proc.pid}")
 
         try:
             comm_q_result = comm_q.get(block=True, timeout=120)
         except queue.Empty:
-            self.logger.error("Hub has not completed initialization in 120s. Aborting")
+            logger.error("Hub has not completed initialization in 120s. Aborting")
             raise Exception("Hub failed to start")
 
         if isinstance(comm_q_result, str):
-            self.logger.error(f"MonitoringRouter sent an error message: {comm_q_result}")
+            logger.error(f"MonitoringRouter sent an error message: {comm_q_result}")
             raise RuntimeError(f"MonitoringRouter failed to start: {comm_q_result}")
 
-        udp_port, ic_port = comm_q_result
+        udp_port, zmq_port = comm_q_result
 
         self.monitoring_hub_url = "udp://{}:{}".format(self.hub_address, udp_port)
 
@@ -217,28 +213,28 @@ class MonitoringHub(RepresentationMixin):
         self._dfk_channel.setsockopt(zmq.LINGER, 0)
         self._dfk_channel.set_hwm(0)
         self._dfk_channel.setsockopt(zmq.SNDTIMEO, self.dfk_channel_timeout)
-        self._dfk_channel.connect("tcp://{}:{}".format(self.hub_address, ic_port))
+        self._dfk_channel.connect("tcp://{}:{}".format(self.hub_address, zmq_port))
 
-        self.logger.info("Monitoring Hub initialized")
+        logger.info("Monitoring Hub initialized")
 
-        return ic_port
+        return zmq_port
 
     # TODO: tighten the Any message format
     def send(self, mtype: MessageType, message: Any) -> None:
-        self.logger.debug("Sending message type {}".format(mtype))
+        logger.debug("Sending message type {}".format(mtype))
         try:
             self._dfk_channel.send_pyobj((mtype, message))
         except zmq.Again:
-            self.logger.exception(
+            logger.exception(
                 "The monitoring message sent from DFK to router timed-out after {}ms".format(self.dfk_channel_timeout))
 
     def close(self) -> None:
-        self.logger.info("Terminating Monitoring Hub")
+        logger.info("Terminating Monitoring Hub")
         exception_msgs = []
         while True:
             try:
                 exception_msgs.append(self.exception_q.get(block=False))
-                self.logger.error("There was a queued exception (Either router or DBM process got exception much earlier?)")
+                logger.error("There was a queued exception (Either router or DBM process got exception much earlier?)")
             except queue.Empty:
                 break
         if self._dfk_channel and self.monitoring_hub_active:
@@ -246,7 +242,7 @@ class MonitoringHub(RepresentationMixin):
             self._dfk_channel.close()
             if exception_msgs:
                 for exception_msg in exception_msgs:
-                    self.logger.error(
+                    logger.error(
                         "{} process delivered an exception: {}. Terminating all monitoring processes immediately.".format(
                             exception_msg[0],
                             exception_msg[1]
@@ -255,21 +251,21 @@ class MonitoringHub(RepresentationMixin):
                 self.router_proc.terminate()
                 self.dbm_proc.terminate()
                 self.filesystem_proc.terminate()
-            self.logger.info("Waiting for router to terminate")
+            logger.info("Waiting for router to terminate")
             self.router_proc.join()
-            self.logger.debug("Finished waiting for router termination")
+            logger.debug("Finished waiting for router termination")
             if len(exception_msgs) == 0:
-                self.logger.debug("Sending STOP to DBM")
+                logger.debug("Sending STOP to DBM")
                 self.priority_msgs.put(("STOP", 0))
             else:
-                self.logger.debug("Not sending STOP to DBM, because there were DBM exceptions")
-            self.logger.debug("Waiting for DB termination")
+                logger.debug("Not sending STOP to DBM, because there were DBM exceptions")
+            logger.debug("Waiting for DB termination")
             self.dbm_proc.join()
-            self.logger.debug("Finished waiting for DBM termination")
+            logger.debug("Finished waiting for DBM termination")
 
             # should this be message based? it probably doesn't need to be if
             # we believe we've received all messages
-            self.logger.info("Terminating filesystem radio receiver process")
+            logger.info("Terminating filesystem radio receiver process")
             self.filesystem_proc.terminate()
             self.filesystem_proc.join()
 
@@ -325,190 +321,3 @@ def filesystem_receiver(logdir: str, q: "queue.Queue[AddressedMonitoringMessage]
                 logger.exception(f"Exception processing {filename} - probably will be retried next iteration")
 
         time.sleep(1)  # whats a good time for this poll?
-
-
-class MonitoringRouter:
-
-    def __init__(self,
-                 *,
-                 hub_address: str,
-                 hub_port: Optional[int] = None,
-                 hub_port_range: Tuple[int, int] = (55050, 56000),
-
-                 monitoring_hub_address: str = "127.0.0.1",
-                 logdir: str = ".",
-                 run_id: str,
-                 logging_level: int = logging.INFO,
-                 atexit_timeout: int = 3    # in seconds
-                 ):
-        """ Initializes a monitoring configuration class.
-
-        Parameters
-        ----------
-        hub_address : str
-             The ip address at which the workers will be able to reach the Hub.
-        hub_port : int
-             The specific port at which workers will be able to reach the Hub via UDP. Default: None
-        hub_port_range : tuple(int, int)
-             The MonitoringHub picks ports at random from the range which will be used by Hub.
-             This is overridden when the hub_port option is set. Default: (55050, 56000)
-        logdir : str
-             Parsl log directory paths. Logs and temp files go here. Default: '.'
-        logging_level : int
-             Logging level as defined in the logging module. Default: logging.INFO
-        atexit_timeout : float, optional
-            The amount of time in seconds to terminate the hub without receiving any messages, after the last dfk workflow message is received.
-
-        """
-        os.makedirs(logdir, exist_ok=True)
-        self.logger = set_file_logger("{}/monitoring_router.log".format(logdir),
-                                      name="monitoring_router",
-                                      level=logging_level)
-        self.logger.debug("Monitoring router starting")
-
-        self.hub_address = hub_address
-        self.atexit_timeout = atexit_timeout
-        self.run_id = run_id
-
-        self.loop_freq = 10.0  # milliseconds
-
-        # Initialize the UDP socket
-        self.sock = socket.socket(socket.AF_INET,
-                                  socket.SOCK_DGRAM,
-                                  socket.IPPROTO_UDP)
-
-        # We are trying to bind to all interfaces with 0.0.0.0
-        if not hub_port:
-            self.sock.bind(('0.0.0.0', 0))
-            self.hub_port = self.sock.getsockname()[1]
-        else:
-            self.hub_port = hub_port
-            try:
-                self.sock.bind(('0.0.0.0', self.hub_port))
-            except Exception as e:
-                raise RuntimeError(f"Could not bind to hub_port {hub_port} because: {e}")
-        self.sock.settimeout(self.loop_freq / 1000)
-        self.logger.info("Initialized the UDP socket on 0.0.0.0:{}".format(self.hub_port))
-
-        self._context = zmq.Context()
-        self.ic_channel = self._context.socket(zmq.DEALER)
-        self.ic_channel.setsockopt(zmq.LINGER, 0)
-        self.ic_channel.set_hwm(0)
-        self.ic_channel.RCVTIMEO = int(self.loop_freq)  # in milliseconds
-        self.logger.debug("hub_address: {}. hub_port_range {}".format(hub_address, hub_port_range))
-        self.ic_port = self.ic_channel.bind_to_random_port("tcp://*",
-                                                           min_port=hub_port_range[0],
-                                                           max_port=hub_port_range[1])
-
-    def start(self,
-              priority_msgs: "queue.Queue[AddressedMonitoringMessage]",
-              node_msgs: "queue.Queue[AddressedMonitoringMessage]",
-              block_msgs: "queue.Queue[AddressedMonitoringMessage]",
-              resource_msgs: "queue.Queue[AddressedMonitoringMessage]") -> None:
-        try:
-            router_keep_going = True
-            while router_keep_going:
-                try:
-                    data, addr = self.sock.recvfrom(2048)
-                    resource_msg = pickle.loads(data)
-                    self.logger.debug("Got UDP Message from {}: {}".format(addr, resource_msg))
-                    resource_msgs.put((resource_msg, addr))
-                except socket.timeout:
-                    pass
-
-                try:
-                    dfk_loop_start = time.time()
-                    while time.time() - dfk_loop_start < 1.0:  # TODO make configurable
-                        # note that nothing checks that msg really is of the annotated type
-                        msg: TaggedMonitoringMessage
-                        msg = self.ic_channel.recv_pyobj()
-
-                        assert isinstance(msg, tuple), "IC Channel expects only tuples, got {}".format(msg)
-                        assert len(msg) >= 1, "IC Channel expects tuples of length at least 1, got {}".format(msg)
-                        assert len(msg) == 2, "IC Channel expects message tuples of exactly length 2, got {}".format(msg)
-
-                        msg_0: AddressedMonitoringMessage
-                        msg_0 = (msg, 0)
-
-                        if msg[0] == MessageType.NODE_INFO:
-                            msg[1]['run_id'] = self.run_id
-                            node_msgs.put(msg_0)
-                        elif msg[0] == MessageType.RESOURCE_INFO:
-                            resource_msgs.put(msg_0)
-                        elif msg[0] == MessageType.BLOCK_INFO:
-                            block_msgs.put(msg_0)
-                        elif msg[0] == MessageType.TASK_INFO:
-                            priority_msgs.put(msg_0)
-                        elif msg[0] == MessageType.WORKFLOW_INFO:
-                            priority_msgs.put(msg_0)
-                            if 'exit_now' in msg[1] and msg[1]['exit_now']:
-                                router_keep_going = False
-                        else:
-                            # There is a type: ignore here because if msg[0]
-                            # is of the correct type, this code is unreachable,
-                            # but there is no verification that the message
-                            # received from ic_channel.recv_pyobj() is actually
-                            # of that type.
-                            self.logger.error("Discarding message "  # type: ignore[unreachable]
-                                              f"from interchange with unknown type {msg[0].value}")
-                except zmq.Again:
-                    pass
-                except Exception:
-                    # This will catch malformed messages. What happens if the
-                    # channel is broken in such a way that it always raises
-                    # an exception? Looping on this would maybe be the wrong
-                    # thing to do.
-                    self.logger.warning("Failure processing a ZMQ message", exc_info=True)
-
-            self.logger.info("Monitoring router draining")
-            last_msg_received_time = time.time()
-            while time.time() - last_msg_received_time < self.atexit_timeout:
-                try:
-                    data, addr = self.sock.recvfrom(2048)
-                    msg = pickle.loads(data)
-                    self.logger.debug("Got UDP Message from {}: {}".format(addr, msg))
-                    resource_msgs.put((msg, addr))
-                    last_msg_received_time = time.time()
-                except socket.timeout:
-                    pass
-
-            self.logger.info("Monitoring router finishing normally")
-        finally:
-            self.logger.info("Monitoring router finished")
-
-
-@wrap_with_logs
-def router_starter(comm_q: "queue.Queue[Union[Tuple[int, int], str]]",
-                   exception_q: "queue.Queue[Tuple[str, str]]",
-                   priority_msgs: "queue.Queue[AddressedMonitoringMessage]",
-                   node_msgs: "queue.Queue[AddressedMonitoringMessage]",
-                   block_msgs: "queue.Queue[AddressedMonitoringMessage]",
-                   resource_msgs: "queue.Queue[AddressedMonitoringMessage]",
-
-                   hub_address: str,
-                   hub_port: Optional[int],
-                   hub_port_range: Tuple[int, int],
-
-                   logdir: str,
-                   logging_level: int,
-                   run_id: str) -> None:
-    setproctitle("parsl: monitoring router")
-    try:
-        router = MonitoringRouter(hub_address=hub_address,
-                                  hub_port=hub_port,
-                                  hub_port_range=hub_port_range,
-                                  logdir=logdir,
-                                  logging_level=logging_level,
-                                  run_id=run_id)
-    except Exception as e:
-        logger.error("MonitoringRouter construction failed.", exc_info=True)
-        comm_q.put(f"Monitoring router construction failed: {e}")
-    else:
-        comm_q.put((router.hub_port, router.ic_port))
-
-        router.logger.info("Starting MonitoringRouter in router_starter")
-        try:
-            router.start(priority_msgs, node_msgs, block_msgs, resource_msgs)
-        except Exception as e:
-            router.logger.exception("router.start exception")
-            exception_q.put(('Hub', str(e)))

parsl/monitoring/router.py

@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+import os
+import socket
+import time
+import pickle
+import logging
+import zmq
+
+import queue
+
+from parsl.log_utils import set_file_logger
+from parsl.process_loggers import wrap_with_logs
+from parsl.utils import setproctitle
+
+from parsl.monitoring.message_type import MessageType
+from parsl.monitoring.types import AddressedMonitoringMessage, TaggedMonitoringMessage
+from typing import Optional, Tuple, Union
+
+
+logger = logging.getLogger(__name__)
+
+
+class MonitoringRouter:
+
+    def __init__(self,
+                 *,
+                 hub_address: str,
+                 udp_port: Optional[int] = None,
+                 zmq_port_range: Tuple[int, int] = (55050, 56000),
+
+                 monitoring_hub_address: str = "127.0.0.1",
+                 logdir: str = ".",
+                 run_id: str,
+                 logging_level: int = logging.INFO,
+                 atexit_timeout: int = 3    # in seconds
+                 ):
+        """ Initializes a monitoring configuration class.
+
+        Parameters
+        ----------
+        hub_address : str
+             The ip address at which the workers will be able to reach the Hub.
+        udp_port : int
+             The specific port at which workers will be able to reach the Hub via UDP. Default: None
+        zmq_port_range : tuple(int, int)
+             The MonitoringHub picks ports at random from the range which will be used by Hub.
+             Default: (55050, 56000)
+        logdir : str
+             Parsl log directory paths. Logs and temp files go here. Default: '.'
+        logging_level : int
+             Logging level as defined in the logging module. Default: logging.INFO
+        atexit_timeout : float, optional
+            The amount of time in seconds to terminate the hub without receiving any messages, after the last dfk workflow message is received.
+
+        """
+        os.makedirs(logdir, exist_ok=True)
+        self.logger = set_file_logger("{}/monitoring_router.log".format(logdir),
+                                      name="monitoring_router",
+                                      level=logging_level)
+        self.logger.debug("Monitoring router starting")
+
+        self.hub_address = hub_address
+        self.atexit_timeout = atexit_timeout
+        self.run_id = run_id
+
+        self.loop_freq = 10.0  # milliseconds
+
+        # Initialize the UDP socket
+        self.udp_sock = socket.socket(socket.AF_INET,
+                                      socket.SOCK_DGRAM,
+                                      socket.IPPROTO_UDP)
+
+        # We are trying to bind to all interfaces with 0.0.0.0
+        if not udp_port:
+            self.udp_sock.bind(('0.0.0.0', 0))
+            self.udp_port = self.udp_sock.getsockname()[1]
+        else:
+            self.udp_port = udp_port
+            try:
+                self.udp_sock.bind(('0.0.0.0', self.udp_port))
+            except Exception as e:
+                raise RuntimeError(f"Could not bind to udp_port {udp_port} because: {e}")
+        self.udp_sock.settimeout(self.loop_freq / 1000)
+        self.logger.info("Initialized the UDP socket on 0.0.0.0:{}".format(self.udp_port))
+
+        self._context = zmq.Context()
+        self.zmq_receiver_channel = self._context.socket(zmq.DEALER)
+        self.zmq_receiver_channel.setsockopt(zmq.LINGER, 0)
+        self.zmq_receiver_channel.set_hwm(0)
+        self.zmq_receiver_channel.RCVTIMEO = int(self.loop_freq)  # in milliseconds
+        self.logger.debug("hub_address: {}. zmq_port_range {}".format(hub_address, zmq_port_range))
+        self.zmq_receiver_port = self.zmq_receiver_channel.bind_to_random_port("tcp://*",
+                                                                               min_port=zmq_port_range[0],
+                                                                               max_port=zmq_port_range[1])
+
+    def start(self,
+              priority_msgs: "queue.Queue[AddressedMonitoringMessage]",
+              node_msgs: "queue.Queue[AddressedMonitoringMessage]",
+              block_msgs: "queue.Queue[AddressedMonitoringMessage]",
+              resource_msgs: "queue.Queue[AddressedMonitoringMessage]") -> None:
+        try:
+            router_keep_going = True
+            while router_keep_going:
+                try:
+                    data, addr = self.udp_sock.recvfrom(2048)
+                    resource_msg = pickle.loads(data)
+                    self.logger.debug("Got UDP Message from {}: {}".format(addr, resource_msg))
+                    resource_msgs.put((resource_msg, addr))
+                except socket.timeout:
+                    pass
+
+                try:
+                    dfk_loop_start = time.time()
+                    while time.time() - dfk_loop_start < 1.0:  # TODO make configurable
+                        # note that nothing checks that msg really is of the annotated type
+                        msg: TaggedMonitoringMessage
+                        msg = self.zmq_receiver_channel.recv_pyobj()
+
+                        assert isinstance(msg, tuple), "ZMQ Receiver expects only tuples, got {}".format(msg)
+                        assert len(msg) >= 1, "ZMQ Receiver expects tuples of length at least 1, got {}".format(msg)
+                        assert len(msg) == 2, "ZMQ Receiver expects message tuples of exactly length 2, got {}".format(msg)
+
+                        msg_0: AddressedMonitoringMessage
+                        msg_0 = (msg, 0)
+
+                        if msg[0] == MessageType.NODE_INFO:
+                            msg[1]['run_id'] = self.run_id
+                            node_msgs.put(msg_0)
+                        elif msg[0] == MessageType.RESOURCE_INFO:
+                            resource_msgs.put(msg_0)
+                        elif msg[0] == MessageType.BLOCK_INFO:
+                            block_msgs.put(msg_0)
+                        elif msg[0] == MessageType.TASK_INFO:
+                            priority_msgs.put(msg_0)
+                        elif msg[0] == MessageType.WORKFLOW_INFO:
+                            priority_msgs.put(msg_0)
+                            if 'exit_now' in msg[1] and msg[1]['exit_now']:
+                                router_keep_going = False
+                        else:
+                            # There is a type: ignore here because if msg[0]
+                            # is of the correct type, this code is unreachable,
+                            # but there is no verification that the message
+                            # received from zmq_receiver_channel.recv_pyobj() is actually
+                            # of that type.
+                            self.logger.error("Discarding message "  # type: ignore[unreachable]
+                                              f"from interchange with unknown type {msg[0].value}")
+                except zmq.Again:
+                    pass
+                except Exception:
+                    # This will catch malformed messages. What happens if the
+                    # channel is broken in such a way that it always raises
+                    # an exception? Looping on this would maybe be the wrong
+                    # thing to do.
+                    self.logger.warning("Failure processing a ZMQ message", exc_info=True)
+
+            self.logger.info("Monitoring router draining")
+            last_msg_received_time = time.time()
+            while time.time() - last_msg_received_time < self.atexit_timeout:
+                try:
+                    data, addr = self.udp_sock.recvfrom(2048)
+                    msg = pickle.loads(data)
+                    self.logger.debug("Got UDP Message from {}: {}".format(addr, msg))
+                    resource_msgs.put((msg, addr))
+                    last_msg_received_time = time.time()
+                except socket.timeout:
+                    pass
+
+            self.logger.info("Monitoring router finishing normally")
+        finally:
+            self.logger.info("Monitoring router finished")
+
+
+@wrap_with_logs
+def router_starter(comm_q: "queue.Queue[Union[Tuple[int, int], str]]",
+                   exception_q: "queue.Queue[Tuple[str, str]]",
+                   priority_msgs: "queue.Queue[AddressedMonitoringMessage]",
+                   node_msgs: "queue.Queue[AddressedMonitoringMessage]",
+                   block_msgs: "queue.Queue[AddressedMonitoringMessage]",
+                   resource_msgs: "queue.Queue[AddressedMonitoringMessage]",
+
+                   hub_address: str,
+                   udp_port: Optional[int],
+                   zmq_port_range: Tuple[int, int],
+
+                   logdir: str,
+                   logging_level: int,
+                   run_id: str) -> None:
+    setproctitle("parsl: monitoring router")
+    try:
+        router = MonitoringRouter(hub_address=hub_address,
+                                  udp_port=udp_port,
+                                  zmq_port_range=zmq_port_range,
+                                  logdir=logdir,
+                                  logging_level=logging_level,
+                                  run_id=run_id)
+    except Exception as e:
+        logger.error("MonitoringRouter construction failed.", exc_info=True)
+        comm_q.put(f"Monitoring router construction failed: {e}")
+    else:
+        comm_q.put((router.udp_port, router.zmq_receiver_port))
+
+        router.logger.info("Starting MonitoringRouter in router_starter")
+        try:
+            router.start(priority_msgs, node_msgs, block_msgs, resource_msgs)
+        except Exception as e:
+            router.logger.exception("router.start exception")
+            exception_q.put(('Hub', str(e)))

parsl/tests/test_monitoring/test_fuzz_zmq.py

@@ -41,11 +41,11 @@ def test_row_counts():
 
     # dig out the interchange port...
     hub_address = parsl.dfk().hub_address
-    hub_interchange_port = parsl.dfk().hub_interchange_port
+    hub_zmq_port = parsl.dfk().hub_zmq_port
 
     # this will send a string to a new socket connection
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.connect((hub_address, hub_interchange_port))
+        s.connect((hub_address, hub_zmq_port))
         s.sendall(b'fuzzing\r')
 
     # this will send a non-object down the DFK's existing ZMQ connection

parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py

@@ -0,0 +1,82 @@
+import logging
+import os
+import parsl
+import pytest
+import time
+
+from parsl.providers import LocalProvider
+from parsl.channels import LocalChannel
+from parsl.launchers import SimpleLauncher
+
+from parsl.config import Config
+from parsl.executors import HighThroughputExecutor
+from parsl.monitoring import MonitoringHub
+
+
+def fresh_config(run_dir, strategy, db_url):
+    return Config(
+        run_dir=os.fspath(run_dir),
+        executors=[
+            HighThroughputExecutor(
+                label="htex_local",
+                cores_per_worker=1,
+                encrypted=True,
+                provider=LocalProvider(
+                    channel=LocalChannel(),
+                    init_blocks=1,
+                    # min and max are set to 0 to ensure that we don't get
+                    # a block from ongoing strategy scaling, only from
+                    # init_blocks
+                    min_blocks=0,
+                    max_blocks=0,
+                    launcher=SimpleLauncher(),
+                ),
+            )
+        ],
+        strategy=strategy,
+        strategy_period=0.1,
+        monitoring=MonitoringHub(
+                        hub_address="localhost",
+                        hub_port=55055,
+                        logging_endpoint=db_url
+        )
+    )
+
+
+@parsl.python_app
+def this_app():
+    pass
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("strategy", ('none', 'simple', 'htex_auto_scale'))
+def test_row_counts(tmpd_cwd, strategy):
+    # this is imported here rather than at module level because
+    # it isn't available in a plain parsl install, so this module
+    # would otherwise fail to import and break even a basic test
+    # run.
+    import sqlalchemy
+    from sqlalchemy import text
+
+    db_url = f"sqlite:///{tmpd_cwd}/monitoring.db"
+    parsl.load(fresh_config(tmpd_cwd, strategy, db_url))
+
+    this_app().result()
+
+    parsl.dfk().cleanup()
+    parsl.clear()
+
+    engine = sqlalchemy.create_engine(db_url)
+    with engine.begin() as connection:
+
+        result = connection.execute(text("SELECT COUNT(DISTINCT block_id) FROM block"))
+        (c, ) = result.first()
+        assert c == 1, "We should see a single block in this database"
+
+        result = connection.execute(text("SELECT COUNT(*) FROM block WHERE block_id = 0 AND status = 'PENDING'"))
+        (c, ) = result.first()
+        assert c == 1, "There should be a single pending status"
+
+        result = connection.execute(text("SELECT COUNT(*) FROM block WHERE block_id = 0 AND status = 'CANCELLED'"))
+        (c, ) = result.first()
+        assert c == 1, "There should be a single cancelled status"

parsl/tests/test_python_apps/test_context_manager.py

@@ -0,0 +1,40 @@
+import parsl
+from parsl.tests.configs.local_threads import fresh_config
+import pytest
+from parsl.errors import NoDataFlowKernelError
+
+
+@parsl.python_app
+def square(x):
+    return x * x
+
+
+@parsl.bash_app
+def foo(x, stdout='foo.stdout'):
+    return f"echo {x + 1}"
+
+
+def local_setup():
+    pass
+
+
+def local_teardown():
+    parsl.clear()
+
+
+@pytest.mark.local
+def test_within_context_manger():
+    config = fresh_config()
+    with parsl.load(config=config):
+        py_future = square(2)
+        assert py_future.result() == 4
+
+        bash_future = foo(1)
+        assert bash_future.result() == 0
+
+        with open('foo.stdout', 'r') as f:
+            assert f.read() == "2\n"
+
+    with pytest.raises(NoDataFlowKernelError) as excinfo:
+        square(2).result()
+    assert str(excinfo.value) == "Cannot submit to a DFK that has been cleaned up"

parsl/tests/test_shutdown/test_kill_monitoring.py

@@ -0,0 +1,65 @@
+import os
+import parsl
+import pytest
+import signal
+import time
+
+from parsl.tests.configs.htex_local_alternate import fresh_config
+
+# This is a very generous upper bound on expected shutdown time of target
+# process after receiving a signal, measured in seconds.
+PERMITTED_SHUTDOWN_TIME_S = 60
+
+
+@parsl.python_app
+def simple_app():
+    return True
+
+
+@pytest.mark.local
+def test_no_kills():
+    """This tests that we can create a monitoring-enabled DFK and shut it down."""
+
+    parsl.load(fresh_config())
+
+    assert parsl.dfk().monitoring is not None, "This test requires monitoring"
+
+    parsl.dfk().cleanup()
+    parsl.clear()
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("sig", [signal.SIGINT, signal.SIGTERM, signal.SIGKILL, signal.SIGQUIT])
+@pytest.mark.parametrize("process_attr", ["router_proc", "dbm_proc"])
+def test_kill_monitoring_helper_process(sig, process_attr, try_assert):
+    """This tests that we can kill a monitoring process and still have successful shutdown.
+    SIGINT emulates some racy behaviour when ctrl-C is pressed: that
+    monitoring processes receive a ctrl-C too, and so the other processes
+    need to be tolerant to monitoring processes arbitrarily exiting.
+    """
+
+    parsl.load(fresh_config())
+
+    dfk = parsl.dfk()
+
+    assert dfk.monitoring is not None, "Monitoring required"
+
+    target_proc = getattr(dfk.monitoring, process_attr)
+
+    assert target_proc is not None, "prereq: target process must exist"
+    assert target_proc.is_alive(), "prereq: target process must be alive"
+
+    target_pid = target_proc.pid
+    assert target_pid is not None, "prereq: target process must have a pid"
+
+    os.kill(target_pid, sig)
+
+    try_assert(lambda: not target_proc.is_alive(), timeout_ms=PERMITTED_SHUTDOWN_TIME_S * 1000)
+
+    # now we have broken one piece of the monitoring system, do some app
+    # execution and then shut down.
+
+    simple_app().result()
+
+    parsl.dfk().cleanup()
+    parsl.clear()

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.03.18'
+VERSION = '2024.03.25'

{parsl-2024.3.18.dist-info → parsl-2024.3.25.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.3.18
+Version: 2024.3.25
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.03.18.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.03.25.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0
@@ -54,7 +54,7 @@ Requires-Dist: pyyaml ; extra == 'all'
 Requires-Dist: cffi ; extra == 'all'
 Requires-Dist: jsonschema ; extra == 'all'
 Requires-Dist: proxystore ; extra == 'all'
-Requires-Dist: radical.pilot ; extra == 'all'
+Requires-Dist: radical.pilot ==1.47 ; extra == 'all'
 Provides-Extra: aws
 Requires-Dist: boto3 ; extra == 'aws'
 Provides-Extra: azure
@@ -83,7 +83,7 @@ Requires-Dist: oauth-ssh >=0.9 ; extra == 'oauth_ssh'
 Provides-Extra: proxystore
 Requires-Dist: proxystore ; extra == 'proxystore'
 Provides-Extra: radical-pilot
-Requires-Dist: radical.pilot ; extra == 'radical-pilot'
+Requires-Dist: radical.pilot ==1.47 ; extra == 'radical-pilot'
 Provides-Extra: visualization
 Requires-Dist: pydot ; extra == 'visualization'
 Requires-Dist: networkx <2.6,>=2.5 ; extra == 'visualization'

{parsl-2024.3.18.dist-info → parsl-2024.3.25.dist-info}/RECORD

@@ -8,7 +8,7 @@ parsl/multiprocessing.py,sha256=hakfdg-sgxEjwloZeDrt6EhzwdzecvjJhkPHHxh8lII,1938
 parsl/process_loggers.py,sha256=1G3Rfrh5wuZNo2X03grG4kTYPGOxz7hHCyG6L_A3b0A,1137
 parsl/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/utils.py,sha256=A3WDMGaNB4ajVx_jCuc-74W6PFy4zswJy-pLE7u8Dz0,10979
-parsl/version.py,sha256=dhmXAlQKRlQPsExiUS22Ca2clSFcL-0irfA1G-7kIyo,131
+parsl/version.py,sha256=xw1wZ0QmQ9UaGCS5xiCsTOYpLaH5Ht4qp-xYq_4FbaE,131
 parsl/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/app/app.py,sha256=wAHchJetgnicT1pn0NJKDeDX0lV3vDFlG8cQd_Ciax4,8522
 parsl/app/bash.py,sha256=bx9x1XFwkOTpZZD3CPwnVL9SyNRDjbUGtOnuGLvxN_8,5396
@@ -60,7 +60,7 @@ parsl/data_provider/http.py,sha256=nDHTW7XmJqAukWJjPRQjyhUXt8r6GsQ36mX9mv_wOig,2
 parsl/data_provider/rsync.py,sha256=2-ZxqrT-hBj39x082NusJaBqsGW4Jd2qCW6JkVPpEl0,4254
 parsl/data_provider/staging.py,sha256=l-mAXFburs3BWPjkSmiQKuAgJpsxCG62yATPDbrafYI,4523
 parsl/dataflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/dataflow/dflow.py,sha256=1xvZKXDleb1GMS7x4dg8BGhKJfD0wPq4bjbAtFbkkc4,64256
+parsl/dataflow/dflow.py,sha256=38bZ0cFnCKqlbyf-XYxyA8PscsypQNOzd_8bIUWT0wM,64461
 parsl/dataflow/errors.py,sha256=w2vOt_ymzG2dOqJUO4IDcmTlrCIHlMZL8nBVyVq0O_8,2176
 parsl/dataflow/futures.py,sha256=aVfEUTzp4-EdunDAtNcqVQf8l_A7ArDi2c82KZMwxfY,5256
 parsl/dataflow/memoization.py,sha256=AsJO6c6cRp2ac6H8uGn2USlEi78_nX3QWvpxYt4XdYE,9583
@@ -68,9 +68,9 @@ parsl/dataflow/rundirs.py,sha256=XKmBZpBEIsGACBhYOkbbs2e5edC0pQegJcSlk4FWeag,115
 parsl/dataflow/states.py,sha256=hV6mfv-y4A6xrujeQglcomnfEs7y3Xm2g6JFwC6dvgQ,2612
 parsl/dataflow/taskrecord.py,sha256=bzIBmlDTsRrELtB9PUQwxTWcwrCd8aMsUAzvijle1eo,3114
 parsl/executors/__init__.py,sha256=J50N97Nm9YRjz6K0oNXDxUYIsDjL43_tp3LVb2w7n-M,381
-parsl/executors/base.py,sha256=DyKzXZztPagh9xQykTUuJRMR9g3i6qkbZi8YpWrIfXM,4340
+parsl/executors/base.py,sha256=CNWddQ7eP_Kqd2THv4bj5Dg1Jgb3dbI3z3aznsRP6dc,4574
 parsl/executors/errors.py,sha256=xVswxgi7vmJcUMCeYDAPK8sQT2kHFFROVoOr0dnmcWE,2098
-parsl/executors/status_handling.py,sha256=GNBYrAhOpHSnmJA0NouK9by_uJxXn8vlPgBhcp0pSFo,10940
+parsl/executors/status_handling.py,sha256=DP7Wu2BjoMPcJ8bdYmwALHvC5Fd1y8-g9sLj_cKoGOQ,10574
 parsl/executors/threads.py,sha256=bMU3JFghm17Lpcua13pr3NgQhkUDDc2mqvF2yJBrVNQ,3353
 parsl/executors/flux/__init__.py,sha256=P9grTTeRPXfqXurFhlSS7XhmE6tTbnCnyQ1f9b-oYHE,136
 parsl/executors/flux/execute_parsl_task.py,sha256=yUG_WjZLcX8LrgPl26mpEBWZhQMlVNbRLGu08yIjdf4,1553
@@ -95,7 +95,7 @@ parsl/executors/radical/rpex_worker.py,sha256=1M1df-hzFdmZMWbRZlUzIX7uAWMKJ_SkxL
 parsl/executors/taskvine/__init__.py,sha256=sWIJdvSLgQKul9dlSjIkNat7yBDgU3SrBF3X2yhT86E,293
 parsl/executors/taskvine/errors.py,sha256=MNS_NjpvHjwevQXOjqjSEBFroqEWi-LT1ZEVZ2C5Dx0,652
 parsl/executors/taskvine/exec_parsl_function.py,sha256=oUAKbPWwpbzWwQ47bZQlVDxS8txhnhPsonMf3AOEMGQ,7085
-parsl/executors/taskvine/executor.py,sha256=KHiRGcLWHyVBm_6E4IowCV20Z15lsf2ZUnTR49IQxNg,31544
+parsl/executors/taskvine/executor.py,sha256=YAPnZZV31R_H1A4mILNIiDQVVvzO6G1wUo0HIjmcw7g,32264
 parsl/executors/taskvine/factory.py,sha256=sHhfGv7xRFrWkQclzRXuFEAHuSXhsZu2lR5LJ81aucA,2638
 parsl/executors/taskvine/factory_config.py,sha256=AbE2fN2snrF5ITYrrS4DnGn2XkJHUFr_17DYHDHIwq0,3693
 parsl/executors/taskvine/manager.py,sha256=VxVN2L5zFVPNfSAJrGgq87MRJKpcxf-BHdO5QWxB4TU,25822
@@ -104,13 +104,13 @@ parsl/executors/taskvine/utils.py,sha256=iSrIogeiauL3UNy_9tiZp1cBSNn6fIJkMYQRVi1
 parsl/executors/workqueue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/executors/workqueue/errors.py,sha256=ghB93Ptb_QbOAvgLe7siV_snRRkU_T-cFHv3AR6Ziwo,541
 parsl/executors/workqueue/exec_parsl_function.py,sha256=NtWNeBvRqksej38eRPw8zPBJ1CeW6vgaitve0tfz_qc,7801
-parsl/executors/workqueue/executor.py,sha256=En8UJJbcQUY88bwYchASco71LTWb6YfXXlbxPlD46Sk,49107
+parsl/executors/workqueue/executor.py,sha256=QnHNdj7aVVYOzK-jmo0YqKMqW2__XmmruHHilqGUVy0,49823
 parsl/executors/workqueue/parsl_coprocess.py,sha256=kEFGC-A97c_gweUPvrc9EEGume7vUpkJLJlyAb87xtQ,5737
 parsl/executors/workqueue/parsl_coprocess_stub.py,sha256=_bJmpPIgL42qM6bVzeEKt1Mn1trSP41rtJguXxPGfHI,735
 parsl/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-parsl/jobs/error_handlers.py,sha256=dvfZFqTwAcFXrIx3sjFTcjtwOB0-xGn6QnPwJEG-kAI,2311
+parsl/jobs/error_handlers.py,sha256=WcWZUA7KyE1ocX5zrMf_EwqOob8Jb7uHMjD3nlb_BUo,2319
 parsl/jobs/errors.py,sha256=cpSQXCrlKtuHsQf7usjF-lX8XsDkFnE5kWpmFjiN6OU,178
-parsl/jobs/job_status_poller.py,sha256=30W-KvzYGQfblFwe_6ZpRVvzerwZD76IC4Xvt9FBw1c,5349
+parsl/jobs/job_status_poller.py,sha256=hLFyT9tnvYrIev72bg0jZUhabYR9GJ0o15M5l9TR3Qo,5422
 parsl/jobs/states.py,sha256=rPBoAEEudKngWFijlwvXXhAagDs_9DCXvQP9rwzVgCM,4855
 parsl/jobs/strategy.py,sha256=a-W3vxEHHCfe521LMfSoZLpJjdTtwCfTgdn1ChxzUuI,12959
 parsl/launchers/__init__.py,sha256=k8zAB3IBP-brfqXUptKwGkvsIRaXjAJZNBJa2XVtY1A,546
@@ -120,9 +120,10 @@ parsl/launchers/launchers.py,sha256=VB--fiVv_IQne3DydTMSdGUY0o0g69puAs-Hd3mJ2vo,
 parsl/monitoring/__init__.py,sha256=0ywNz6i0lM1xo_7_BIxhETDGeVd2C_0wwD7qgeaMR4c,83
 parsl/monitoring/db_manager.py,sha256=hdmmXSTXp8Wwhr7vLpQalD_ahRl3SNxKYVsplnThRk8,37021
 parsl/monitoring/message_type.py,sha256=Khn88afNxcOIciKiCK4GLnn90I5BlRTiOL3zK-P07yQ,401
-parsl/monitoring/monitoring.py,sha256=N_g3ijGJK46y0k2CEWMGvGR4Ym1mR13yeBkW6TiHHMo,23488
+parsl/monitoring/monitoring.py,sha256=ZkwbIKGG7Zx8Nxj8TEaTT_FrmjwJvmv1jNlEmaUZYAM,14313
 parsl/monitoring/radios.py,sha256=T2_6QuUjC-dd_7qMnIk6WHQead1iWz7m_P6ZC4QAqdA,5265
 parsl/monitoring/remote.py,sha256=OcIgudujtPO_DsY-YV36x92skeiNdGt-6aEOqaCU8T0,13900
+parsl/monitoring/router.py,sha256=Y_PJjffS23HwfTJClhg5W4gUXnkAI_3crjjZMoyzxVA,9592
 parsl/monitoring/types.py,sha256=SO6Fjjbb83sv_MtbutoxGssiWh6oXKkEEsD4EvwOnZ4,629
 parsl/monitoring/queries/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/monitoring/queries/pandas.py,sha256=J09lIxSvVtAfBTbFcwKgUkHwZfaS6vYOzXHxS7UBLoc,2233
@@ -351,7 +352,8 @@ parsl/tests/test_monitoring/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 parsl/tests/test_monitoring/test_app_names.py,sha256=4Ziggxv0JLP0UGAd5jjXdivUdZQLlMvVVMfiTStjxRk,2191
 parsl/tests/test_monitoring/test_basic.py,sha256=uXWx2O2Y2gfSO4e8zTjyj5bucKHG9OVzMxQNnq9abeY,2776
 parsl/tests/test_monitoring/test_db_locks.py,sha256=PGoRmvqA6AYPXTPHOZPLH38Z4D6EEgSb6ZgNfZtwIGk,2910
-parsl/tests/test_monitoring/test_fuzz_zmq.py,sha256=Xi08Drt_DZKbo3Ihl83cgsTTMRok1bBpyrVOQQWlbf4,3145
+parsl/tests/test_monitoring/test_fuzz_zmq.py,sha256=CpTRF2z2xpshlHHTNiNIIJMOx8bxSmSyAwbMYcOkgBk,3121
+parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py,sha256=aFxQWL0p3kmVWHgLjYTx0MdY3JMkVPQSae3pmRn5T3s,2605
 parsl/tests/test_monitoring/test_incomplete_futures.py,sha256=9lJhkWlVB8gCCTkFjObzoh1uCL1pRmU6gFgEzLCztnY,2021
 parsl/tests/test_monitoring/test_memoization_representation.py,sha256=tErT7zseSMaQ5eNmK3hH90J6OZKuAaFQG50OXK6Jy9s,2660
 parsl/tests/test_monitoring/test_viz_colouring.py,sha256=k8SiELxPtnGYZ4r02VQt46RC61fGDVC4nmY768snX1U,591
@@ -372,6 +374,7 @@ parsl/tests/test_providers/test_submiterror_deprecation.py,sha256=ZutVj_0VJ7M-5U
 parsl/tests/test_python_apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/tests/test_python_apps/test_arg_input_types.py,sha256=JXpfHiu8lr9BN6u1OzqFvGwBhxzsGTPMewHx6Wdo-HI,670
 parsl/tests/test_python_apps/test_basic.py,sha256=lFqh4ugePbp_FRiHGUXxzV34iS7l8C5UkxTHuLcpnYs,855
+parsl/tests/test_python_apps/test_context_manager.py,sha256=ajG8Vy0gxXzMO_rAm7yO_WO6XJCjEqUgmdRHdPBLCY0,844
 parsl/tests/test_python_apps/test_dep_standard_futures.py,sha256=BloeaYBci0jS5al2d8Eqe3OfZ1tvolA5ZflOBQPR9Wo,859
 parsl/tests/test_python_apps/test_dependencies.py,sha256=IRiTI_lPoWBSFSFnaBlE6Bv08PKEaf-qj5dfqO2RjT0,272
 parsl/tests/test_python_apps/test_depfail_propagation.py,sha256=3q3HlVWrOixFtXWBvR_ypKtbdAHAJcKndXQ5drwrBQU,1488
@@ -421,6 +424,8 @@ parsl/tests/test_serialization/test_htex_code_cache.py,sha256=YW9ab4GCpeZWRdYsVE
 parsl/tests/test_serialization/test_pack_resource_spec.py,sha256=eZ_gykB4uTDyEEF1HkExTUn98j9pTljxAnn6ucFhdvo,640
 parsl/tests/test_serialization/test_proxystore_configured.py,sha256=_JbMzeUgcR-1Ss2hGAb2v0LBA0fzKpNpfO-HaUCR7Yo,2293
 parsl/tests/test_serialization/test_proxystore_impl.py,sha256=Pn_4ulwCd7Tc6Qlmypq2ImT4DtErGDIfqHHmPTr7aOI,1226
+parsl/tests/test_shutdown/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+parsl/tests/test_shutdown/test_kill_monitoring.py,sha256=5uzhcOPyvS9u4xzlweC5jV2Inl7eWZ1zFzcev73LWwY,1941
 parsl/tests/test_staging/__init__.py,sha256=WZl9EHSkfYiSoE3Gbulcq2ifmn7IFGUkasJIobL5T5A,208
 parsl/tests/test_staging/staging_provider.py,sha256=DEONOOGrYgDZimE3rkrbzFl_4KxhzmNJledh8Hho9fo,3242
 parsl/tests/test_staging/test_1316.py,sha256=pj1QbmOJSRES1R4Ov380MmVe6xXvPUXh4FB48nE6vjI,2687
@@ -438,12 +443,12 @@ parsl/tests/test_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 parsl/tests/test_utils/test_representation_mixin.py,sha256=kUZeIDwA2rlbJ3-beGzLLwf3dOplTMCrWJN87etHcyY,1633
 parsl/usage_tracking/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 parsl/usage_tracking/usage.py,sha256=pSADeogWqvkYI_n2pojv4IWDEFAQ3KwXNx6LDTohMHQ,6684
-parsl-2024.3.18.data/scripts/exec_parsl_function.py,sha256=NtWNeBvRqksej38eRPw8zPBJ1CeW6vgaitve0tfz_qc,7801
-parsl-2024.3.18.data/scripts/parsl_coprocess.py,sha256=Y7Tc-h9WGui-YDe3w_h91w2Sm1JNL1gJ9kAV4PE_gw8,5722
-parsl-2024.3.18.data/scripts/process_worker_pool.py,sha256=iVrw160CpTAVuX9PH-ezU4ebm9C1_U6IMrkcdyTQJ58,41095
-parsl-2024.3.18.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
-parsl-2024.3.18.dist-info/METADATA,sha256=HywLgkSikpur2yGd42LiMsFytWwnayht8sQasJ6n5j8,3960
-parsl-2024.3.18.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-parsl-2024.3.18.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
-parsl-2024.3.18.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
-parsl-2024.3.18.dist-info/RECORD,,
+parsl-2024.3.25.data/scripts/exec_parsl_function.py,sha256=NtWNeBvRqksej38eRPw8zPBJ1CeW6vgaitve0tfz_qc,7801
+parsl-2024.3.25.data/scripts/parsl_coprocess.py,sha256=Y7Tc-h9WGui-YDe3w_h91w2Sm1JNL1gJ9kAV4PE_gw8,5722
+parsl-2024.3.25.data/scripts/process_worker_pool.py,sha256=iVrw160CpTAVuX9PH-ezU4ebm9C1_U6IMrkcdyTQJ58,41095
+parsl-2024.3.25.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
+parsl-2024.3.25.dist-info/METADATA,sha256=9FTVzNNJqpfn1NqveTatneCqEQhMu3Oo9kA-M7fx40k,3974
+parsl-2024.3.25.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+parsl-2024.3.25.dist-info/entry_points.txt,sha256=XqnsWDYoEcLbsMcpnYGKLEnSBmaIe1YoM5YsBdJG2tI,176
+parsl-2024.3.25.dist-info/top_level.txt,sha256=PIheYoUFQtF2icLsgOykgU-Cjuwr2Oi6On2jo5RYgRM,6
+parsl-2024.3.25.dist-info/RECORD,,